From 0d6153db127133e59137f4359975bafd705e0f4b Mon Sep 17 00:00:00 2001
From: Markus Heiser <markus.heiser@darmarit.de>
Date: Tue, 11 Feb 2020 15:57:42 +0100
Subject: [PATCH] filtron.sh: updated rules from production

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
---
 utils/templates/etc/filtron/rules.json | 226 ++++++++++++++-----------
 1 file changed, 123 insertions(+), 103 deletions(-)

diff --git a/utils/templates/etc/filtron/rules.json b/utils/templates/etc/filtron/rules.json
index 1c7005ae5..f4a99b3e8 100644
--- a/utils/templates/etc/filtron/rules.json
+++ b/utils/templates/etc/filtron/rules.json
@@ -1,105 +1,125 @@
 [
-  { "name": "suspiciously frequent IP",
-    "filters": [],
-    "interval": 600,
-    "limit": 30,
-    "aggregations": [
-      "Header:X-Forwarded-For"
-    ],
-    "actions":[
-      {"name":"log"}
-    ]
-  },
-  { "name": "search request",
-    "filters": [
-      "Param:q",
-      "Path=^(/|/search)$"
-    ],
-    "interval": 61,
-    "limit": 999,
-    "subrules": [
-      {
-        "name": "roboagent limit",
-        "interval": 61,
-        "limit": 1,
-        "filters": [
-          "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client)"
-        ],
-        "actions": [
-          { "name": "log"},
-          { "name": "block",
-            "params": {
-              "message": "Rate limit exceeded"
-            }
-          }
-        ]
-      },
-      {
-        "name": "botlimit",
-        "limit": 0,
-        "stop": true,
-        "filters": [
-          "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
-        ],
-        "actions": [
-          { "name": "log"},
-          { "name": "block",
-            "params": {
-              "message": "Rate limit exceeded"
-            }
-          }
-        ]
-      },
-      {
-        "name": "IP limit",
-        "interval": 61,
-        "limit": 9,
-        "stop": true,
-        "aggregations": [
-          "Header:X-Forwarded-For"
-        ],
-        "actions": [
-          { "name": "log"},
-          { "name": "block",
-            "params": {
-              "message": "Rate limit exceeded"
-            }
-          }
-        ]
-      },
-      {
-        "name": "rss/json limit",
-        "interval": 121,
-        "limit": 2,
-        "stop": true,
-        "filters": [
-          "Param:format=(csv|json|rss)"
-        ],
-        "actions": [
-          { "name": "log"},
-          { "name": "block",
-            "params": {
-              "message": "Rate limit exceeded"
-            }
-          }
-        ]
-      },
-      {
-        "name": "useragent limit",
-        "interval": 61,
-        "limit": 199,
-        "aggregations": [
-          "Header:User-Agent"
-        ],
-        "actions": [
-          { "name": "log"},
-          { "name": "block",
-            "params": {
-              "message": "Rate limit exceeded"
-            }
-          }
-        ]
-      }
-    ]
-  }
+    {
+	"name": "roboagent limit",
+	"filters": [
+	    "Header:User-Agent=(curl|cURL|Wget|python-requests|Scrapy|FeedFetcher|Go-http-client|Ruby|UniversalFeedParser)"
+	],
+	"limit": 0,
+	"stop": true,
+	"actions": [
+	    { "name": "log"},
+	    { "name": "block",
+              "params": {
+		  "message": "Rate limit exceeded"
+              }
+	    }
+	]
+    },
+    {
+	"name": "botlimit",
+	"filters": [
+	    "Header:User-Agent=(Googlebot|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT)"
+	],
+	"limit": 0,
+	"stop": true,
+	"actions": [
+	    { "name": "log"},
+	    { "name": "block",
+              "params": {
+		  "message": "Rate limit exceeded"
+              }
+	    }
+	]
+    },
+    { "name": "suspiciously frequent IP",
+      "filters": [],
+      "interval": 600,
+      "limit": 30,
+      "aggregations": [
+	  "Header:X-Forwarded-For"
+      ],
+      "actions":[
+	  {"name":"log"}
+      ]
+    },
+    { "name": "search request",
+      "filters": [
+	  "Param:q",
+	  "Path=^(/|/search)$"
+      ],
+      "interval": 61,
+      "limit": 999,
+      "subrules": [
+	  {
+	      "name": "missing Accept-Language",
+	      "filters": ["!Header:Accept-Language"],
+	      "limit": 0,
+	      "stop": true,
+	      "actions": [
+		  {"name": "block",
+		   "params": {"message": "Rate limit exceeded"}}
+	      ]
+	  },
+	  {
+              "name": "suspiciously Connection=close header",
+              "filters": ["Header:Connection=close"],
+              "limit": 0,
+              "stop": true,
+              "actions": [
+		  {"name": "block",
+		   "params": {"message": "Rate limit exceeded"}}
+              ]
+	  },
+	  {
+              "name": "IP limit",
+              "interval": 61,
+              "limit": 9,
+              "stop": true,
+              "aggregations": [
+		  "Header:X-Forwarded-For"
+              ],
+              "actions": [
+		  { "name": "log"},
+		  { "name": "block",
+		    "params": {
+			"message": "Rate limit exceeded"
+		    }
+		  }
+              ]
+	  },
+	  {
+              "name": "rss/json limit",
+              "filters": [
+		  "Param:format=(csv|json|rss)"
+              ],
+              "interval": 121,
+              "limit": 2,
+              "stop": true,
+              "actions": [
+		  { "name": "log"},
+		  { "name": "block",
+		    "params": {
+			"message": "Rate limit exceeded"
+		    }
+		  }
+              ]
+	  },
+	  {
+              "name": "useragent limit",
+              "interval": 61,
+              "limit": 199,
+              "aggregations": [
+		  "Header:User-Agent"
+              ],
+              "actions": [
+		  { "name": "log"},
+		  { "name": "block",
+		    "params": {
+			"message": "Rate limit exceeded"
+		    }
+		  }
+              ]
+	  }
+      ]
+    }
 ]