From 27506b7e75927e5dd09761f5eed058580b822771 Mon Sep 17 00:00:00 2001
From: James Moger <james.moger@gitblit.com>
Date: Fri, 15 Jun 2012 08:16:50 -0400
Subject: [PATCH] Configurable robots.txt (issue 99)
---
src/com/gitblit/RobotsTxtServlet.java | 68 ++++++++++++++++++++++++++++++++++
src/WEB-INF/web.xml | 15 +++++++
distrib/gitblit.properties | 7 +++
3 files changed, 89 insertions(+), 1 deletions(-)
diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 440414e..91b1c2b 100644
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -295,6 +295,13 @@
# SINCE 0.7.0
web.enableRpcAdministration = false
+# Full path to a configurable robots.txt file. With this file you can control
+# what parts of your Gitblit server respectable robots are allowed to traverse.
+# http://googlewebmastercentral.blogspot.com/2008/06/improving-on-robots-exclusion-protocol.html
+#
+# SINCE 1.0.0
+web.robots.txt =
+
# If true, the web ui layout will respond and adapt to the browser's dimensions.
# if false, the web ui will use a 940px fixed-width layout.
# http://twitter.github.com/bootstrap/scaffolding.html#responsive
diff --git a/src/WEB-INF/web.xml b/src/WEB-INF/web.xml
index eef49d4..85b24d5 100644
--- a/src/WEB-INF/web.xml
+++ b/src/WEB-INF/web.xml
@@ -99,6 +99,19 @@
</servlet-mapping>
+ <!-- Robots.txt Servlet
+ <url-pattern> MUST match:
+ * Wicket Filter ignorePaths parameter -->
+ <servlet>
+ <servlet-name>RobotsTxtServlet</servlet-name>
+ <servlet-class>com.gitblit.RobotsTxtServlet</servlet-class>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>RobotsTxtServlet</servlet-name>
+ <url-pattern>/robots.txt</url-pattern>
+ </servlet-mapping>
+
+
<!-- Git Access Restriction Filter
<url-pattern> MUST match:
* GitServlet
@@ -202,7 +215,7 @@
* PagesFilter <url-pattern>
* PagesServlet <url-pattern>
* com.gitblit.Constants.PAGES_PATH -->
- <param-value>git/,feed/,zip/,federation/,rpc/,pages/</param-value>
+ <param-value>git/,feed/,zip/,federation/,rpc/,pages/,robots.txt</param-value>
</init-param>
</filter>
<filter-mapping>
diff --git a/src/com/gitblit/RobotsTxtServlet.java b/src/com/gitblit/RobotsTxtServlet.java
new file mode 100644
index 0000000..c142be0
--- /dev/null
+++ b/src/com/gitblit/RobotsTxtServlet.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2012 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import com.gitblit.utils.FileUtils;
+import com.gitblit.utils.StringUtils;
+
+/**
+ * Handles requests for robots.txt
+ *
+ * @author James Moger
+ *
+ */
+public class RobotsTxtServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 1L;
+
+ public RobotsTxtServlet() {
+ super();
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, java.io.IOException {
+ processRequest(request, response);
+ }
+
+ @Override
+ protected void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ processRequest(request, response);
+ }
+
+ protected void processRequest(javax.servlet.http.HttpServletRequest request,
+ javax.servlet.http.HttpServletResponse response) throws javax.servlet.ServletException,
+ java.io.IOException {
+ String robotstxt = GitBlit.getString(Keys.web.robots.txt, null);
+ String content = "";
+ if (!StringUtils.isEmpty(robotstxt)) {
+ File robotsfile = new File(robotstxt);
+ if (robotsfile.exists()) {
+ content = FileUtils.readContent(robotsfile, "\n");
+ }
+ }
+ response.getWriter().append(content);
+ }
+}
--
Gitblit v1.9.1