Merge pull request #330 from mnot/mnot/321

Rough in httplint

.coveragerc

@@ -4,8 +4,8 @@ omit = setup.py
 
 [report]
 exclude_lines =
-	pragma: no cover
-	def __repr__
-	raise AssertionError
-	raise NotImplementedError
-	if __name__ == .__main__.:
+    pragma: no cover
+    def __repr__
+    raise AssertionError
+    raise NotImplementedError
+    if __name__ == .__main__.:

.dockerignore

@@ -24,7 +24,7 @@ README.md
 Dockerfile
 LICENSE
 
-# From .gitingore 
+# From .gitingore
 *.pyc
 __pycache__
 .DS_Store

.editorconfig

@@ -19,3 +19,10 @@ indent_size = 2
 
 [*.html]
 max_line_length = unset
+
+[redbot/assets/**]
+max_line_length = unset
+insert_final_newline = unset
+
+[src/package-lock.json]
+indent_size = 2

.github/workflows/docker-image.yml

@@ -8,7 +8,6 @@ env:
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
-
 jobs:
   publish-docker-image:
 

.github/workflows/test.yml

@@ -42,8 +42,6 @@ jobs:
         key: ${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements.txt') }}
     - name: Set up venv
       run: make venv
-    - name: Check Messages
-      run: make -e message_test
     - name: Typecheck
       run: make typecheck
     - name: Lint

CODE_OF_CONDUCT.md

@@ -23,13 +23,13 @@ include:
 Examples of unacceptable behavior by participants include:
 
 * The use of sexualized language or imagery and unwelcome sexual attention or
- advances
+  advances
 * Trolling, insulting/derogatory comments, and personal or political attacks
 * Public or private harassment
 * Publishing others' private information, such as a physical or electronic
- address, without explicit permission
+  address, without explicit permission
 * Other conduct which could reasonably be considered inappropriate in a
- professional setting
+  professional setting
 
 ## Our Responsibilities
 

CONTRIBUTING.md

@@ -1,4 +1,3 @@
-
 # Contributing to REDbot
 
 Contributions - in the form of code, bugs, or ideas - are very welcome!

Makefile

@@ -6,8 +6,6 @@ WEBPACK=$(NPX) webpack-cli
 CSSMIN=$(NPX) cssmin
 SASS=$(NPX) node-sass
 
-GITHUB_STEP_SUMMARY ?= throwaway
-
 MODULES = src/node_modules
 JS_ENTRIES = ./src/js/red_script.js ./src/js/red_request.js ./src/js/red_response.js ./src/js/red_response_multi.js
 CSSFILES = redbot/assets/red_style.css $(MODULES)/google-code-prettify/src/prettify.css
@@ -33,40 +31,18 @@ typecheck: typecheck_py
 tidy: tidy_py
 	$(STANDARD) --fix "src/js/*.js"
 
-.PHONY: syntax
-syntax: venv
-	PYTHONPATH=$(VENV) $(VENV)/python redbot/syntax/__init__.py
-
 
 #############################################################################
 ## Tests
 
 .PHONY: test
-test: message_test webui_test
+test: webui_test
 
 .PHONY: webui_test
 webui_test: venv
 	$(VENV)/playwright install chromium
 	PYTHONPATH=.:$(VENV) $(VENV)/python test/test_webui.py
 
-.PHONY: message_test
-message_test: venv
-	PYTHONPATH=.:$(VENV) $(VENV)/pytest --md $(GITHUB_STEP_SUMMARY) redbot/message/*.py redbot/message/headers/*.py
-	rm -f throwaway
-
-#############################################################################
-### Coverage
-
-.PHONY: coverage
-coverage: header_coverage note_coverage
-
-.PHONY: header_coverage
-header_coverage: venv
-	PYTHONPATH=$(VENV) $(VENV)/python test/header_coverage.py test/registries/message-headers.xml
-
-.PHONY: note_coverage
-note_coverage: venv
-	PYTHONPATH=$(VENV) $(VENV)/python test/note_coverage.py
 
 #############################################################################
 ## Local test server / cli
@@ -90,13 +66,6 @@ docker-image:
 docker: docker-image
 	docker run --rm --name redbot -p 8000:8000 redbot
 
-#############################################################################
-## Create new headers
-
-redbot/message/headers/%.py:
-	cp redbot/message/headers/_header.tpl $@
-	sed -i '' -e "s/SHORT_NAME/$*/g" $@
-
 #############################################################################
 ## Assets
 

pyproject.toml

@@ -21,6 +21,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License"
 ]
 dependencies = [
+    "httplint >= 2023.11.1",
     "importlib_resources",
     "Jinja2 >= 3.1.2",
     "markdown >= 3.4.4",
@@ -36,8 +37,7 @@ homepage = "https://redbot.org/project/"
 [project.optional-dependencies]
 systemd = ["cysystemd"]
 dev = ["mypy", "black", "pylint", "pytest", "pytest-md", "validate-pyproject", "build",
-       "playwright", "types-Markdown"]
-
+    "playwright", "types-Markdown"]
 
 [project.scripts]
 redbot = "redbot.cli:main"
@@ -72,7 +72,6 @@ warn_unreachable = true
 strict_optional = false
 show_error_codes = true
 
-
 [tool.pylint.basic]
 function-rgx = "[a-z_][a-z0-9_]{1,30}$"
 variable-rgx = "[a-z_][a-z0-9_]{1,30}$"
@@ -83,7 +82,7 @@ method-rgx = "[a-z_][a-z0-9_]{1,30}$"
 class-rgx = "[a-zA-Z0-9_]+$"
 
 [tool.pylint.messages_control]
-disable = "C0114,C0115,C0116,W0613"
+disable = "C0114,C0115,C0116,W0613,R0903"
 
 [tool.pylint.reports]
 reports = false

redbot/daemon.py

@@ -20,10 +20,11 @@
 
 from importlib_resources import files as resource_files
 
+import httplint
 import thor
 from thor.loop import _loop
 
-from redbot import __version__
+import redbot
 from redbot.type import RawHeaderListType
 from redbot.webui import RedWebUi
 from redbot.webui.saved_tests import clean_saved_tests
@@ -255,7 +256,8 @@ def main() -> None:
         _loop.debug = True
 
     sys.stderr.write(
-        f"Starting on PID {os.getpid()}... (thor {thor.__version__})\n"
+        f"Starting REDbot {redbot.__version__} on PID {os.getpid()}"
+        + f" (thor {thor.__version__}; httplint {httplint.__version__})\n"
         + f"http://{conf['redbot'].get('host', '')}:{conf['redbot']['port']}/\n"
     )
 

redbot/formatter/__init__.py

@@ -112,8 +112,8 @@ def bind_resource(self, display_resource: "HttpResource") -> None:
             if display_resource.request.complete:
                 self.start_output()
             else:
-                display_resource.request.on("headers_available", self.start_output)
-            display_resource.response.on("chunk", self.feed)
+                display_resource.on("response_headers_available", self.start_output)
+            display_resource.response_content_processors.append(self.feed)
             display_resource.on("status", self.status)
             display_resource.on("debug", self.debug)
 
@@ -188,9 +188,9 @@ def relative_time(utime: float, now: float = None, show_sign: int = 1) -> str:
     """
     Given two times, return a string that explains how far apart they are.
     show_sign can be:
-      0 - don't show
-      1 - ago / from now  [DEFAULT]
-      2 - early / late
+        0 - don't show
+        1 - ago / from now  [DEFAULT]
+        2 - early / late
     """
 
     signs = {

redbot/formatter/har.py

@@ -10,8 +10,8 @@
 
 from redbot import __version__
 from redbot.formatter import Formatter
-from redbot.message.headers import StrHeaderListType
 from redbot.resource import HttpResource
+from redbot.type import StrHeaderListType
 
 
 class HarLogDict(TypedDict):
@@ -75,7 +75,7 @@ def add_entry(self, resource: HttpResource, page_ref: int = None) -> None:
         entry = {
             "startedDateTime": isoformat(resource.request.start_time),
             "time": int(
-                (resource.response.complete_time - resource.request.start_time) * 1000
+                (resource.response.finish_time - resource.request.start_time) * 1000
             ),
             "_red_messages": self.format_notes(resource),
         }
@@ -87,7 +87,7 @@ def add_entry(self, resource: HttpResource, page_ref: int = None) -> None:
             "url": resource.request.uri,
             "httpVersion": "HTTP/1.1",
             "cookies": [],
-            "headers": self.format_headers(resource.request.headers),
+            "headers": self.format_headers(resource.request.headers.text),
             "queryString": [],
             "headersSize": -1,
             "bodySize": -1,
@@ -98,16 +98,16 @@ def add_entry(self, resource: HttpResource, page_ref: int = None) -> None:
             "statusText": resource.response.status_phrase,
             "httpVersion": f"HTTP/{resource.response.version}",
             "cookies": [],
-            "headers": self.format_headers(resource.response.headers),
+            "headers": self.format_headers(resource.response.headers.text),
             "content": {
-                "size": resource.response.decoded_len,
-                "compression": resource.response.decoded_len
-                - resource.response.payload_len,
-                "mimeType": resource.response.parsed_headers.get("content-type", ""),
+                "size": resource.response.decoded.length,
+                "compression": resource.response.decoded.length
+                - resource.response.content_length,
+                "mimeType": resource.response.headers.parsed.get("content-type", ""),
             },
-            "redirectURL": resource.response.parsed_headers.get("location", ""),
-            "headersSize": resource.response.header_length,
-            "bodySize": resource.response.payload_len,
+            "redirectURL": resource.response.headers.parsed.get("location", ""),
+            "headersSize": resource.response_header_length,
+            "bodySize": resource.response.content_length,
         }
 
         cache: Dict[None, None] = {}
@@ -120,7 +120,7 @@ def add_entry(self, resource: HttpResource, page_ref: int = None) -> None:
                 (resource.response.start_time - resource.request.start_time) * 1000
             ),
             "receive": int(
-                (resource.response.complete_time - resource.response.start_time) * 1000
+                (resource.response.finish_time - resource.response.start_time) * 1000
             ),
         }
 
@@ -157,7 +157,7 @@ def format_notes(self, resource: HttpResource) -> List[Dict[str, str]]:
                 "subject": note.subject,
                 "category": note.category.name,
                 "level": note.level.name,
-                "summary": note.show_summary(self.lang),
+                "summary": note.summary,
             }
             out.append(msg)
         return out

redbot/formatter/html.py

@@ -8,6 +8,8 @@
 from typing import Any, List, Match, Tuple
 from urllib.parse import urljoin
 
+from httplint import get_field_description
+from httplint.note import Note, levels, categories
 import thor.http.error as httperr
 
 from redbot import __version__
@@ -19,8 +21,6 @@
     NL,
 )
 from redbot.resource import HttpResource, active_check
-from redbot.message.headers import HeaderProcessor
-from redbot.speak import Note, levels, categories
 
 
 class SingleEntryHtmlFormatter(BaseHtmlFormatter):
@@ -90,7 +90,7 @@ def __init__(self, *args: Any, **kw: Any) -> None:
 
     def finish_output(self) -> None:
         self.final_status()
-        media_type = self.resource.response.parsed_headers.get("content-type", [""])[0]
+        media_type = self.resource.response.headers.parsed.get("content-type", [""])[0]
         if self.resource.response.complete:
             validator_link = self.validators.get(media_type, None)
             if validator_link:
@@ -124,7 +124,7 @@ def finish_output(self) -> None:
                 )
             )
         else:
-            http_error = self.resource.response.http_error
+            http_error = self.resource.fetch_error
             if http_error is None:
                 pass  # usually a global timeout...
             elif isinstance(http_error, httperr.HttpError):
@@ -134,15 +134,12 @@ def finish_output(self) -> None:
                     self.error_output(http_error.desc)
             else:
                 raise AssertionError(
-                    f"Unknown incomplete response error {self.resource.response.http_error}"
+                    f"Unknown incomplete response error {self.resource.fetch_error}"
                 )
 
     def format_body_sample(self, resource: HttpResource) -> Markup:
         """show the stored body sample"""
-        if resource.response.status_code == "206":
-            sample = resource.response.payload
-        else:
-            sample = resource.response.decoded_sample
+        sample = b"".join(resource.response_decoded_sample)
         try:
             uni_sample = sample.decode(resource.response.character_encoding, "ignore")
         except (TypeError, LookupError):
@@ -177,7 +174,7 @@ def link_to(matchobj: Match) -> str:
                         )
                     )
         message = ""
-        if not resource.response.decoded_sample_complete:
+        if not resource.response_decoded_complete:
             message = "<p class='btw'>REDbot isn't showing all content, because it's so big!</p>"
         return Markup(f"<pre class='prettyprint'>{safe_sample}</pre>\n{message}")
 
@@ -207,7 +204,7 @@ def format_header(self, header: Tuple[str, str]) -> Markup:
         return Markup(self.header_presenter.show(header[0], header[1]))
 
     def format_header_description(self, header_name: str) -> Markup:
-        description = HeaderProcessor.find_header_handler(header_name).description
+        description = get_field_description(header_name)
         if description:
             return Markup(
                 '<span class="tip">'
@@ -222,8 +219,8 @@ def format_header_description(self, header_name: str) -> Markup:
 class HeaderPresenter:
     """
     Present a HTTP header in the Web UI. By default, it will:
-       - Escape HTML sequences to avoid XSS attacks
-       - Wrap long lines
+        - Escape HTML sequences to avoid XSS attacks
+        - Wrap long lines
     However if a method is present that corresponds to the header's
     field-name, that method will be run instead to represent the value.
     """
@@ -335,7 +332,7 @@ def index_problem(self, problem: Note) -> int:
         return self.problems.index(problem) + 1
 
     def format_note_description(self, header_name: str) -> Markup:
-        description = HeaderProcessor.find_header_handler(header_name).description
+        description = get_field_description(header_name)
         if description:
             return Markup(
                 self._markdown.reset().convert(