Suggestion/git submodules4annotation sources

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "annotations/from-icann"]
+	path = annotations/from-icann
+	url = https://github.com/icann/icann-annotations.git
+[submodule "annotations/wes-hardaker"]
+	path = annotations/wes-hardaker
+	url = https://github.com/hardaker/dns-rfc-annotations.git
+[submodule "annotations/isc"]
+	path = annotations/isc
+	url = https://gitlab.isc.org/isc-projects/rfc-annotations.git

Makefile

@@ -5,7 +5,6 @@ CURRENT_CONTAINER=$(shell docker ps -aq --filter name=${CONTAINERNAME})
 CURRENT_IMAGE=$(shell docker image list --filter reference=${IMAGENAME} -q)
 
 all: folders
-	python3 program/pull_updates.py
 	RFC_INDEX="YES" python3 -u program/main.py
 
 generated-html local-config raw-originals raw-originals/drafts annotations/_generated:
@@ -14,7 +13,6 @@ generated-html local-config raw-originals raw-originals/drafts annotations/_gene
 folders: generated-html local-config raw-originals raw-originals/drafts annotations/_generated
 
 annotations: folders
-	python3 program/pull_updates.py
 	RFC_FETCH_FILES="NO" python3 -u program/main.py
 
 test: tests folders
@@ -54,4 +52,4 @@ docker-remove:
 
 clean: docker-remove
 	rm -rf generated-html/rfc*.html && rm -rf raw-originals && rm -rf annotations/_generated && rm -rf .pytest_cache \
-	rm -rf program/__pycache__
+	rm -rf program/__pycache__

README.md

@@ -105,9 +105,10 @@ Annotations that are automatically generated by the tool are put in `annotations
 it is unlikely that you want to touch that folder.
 You can add local annotations by adding subdirectories to the `annotations/` directory.
 
-The tool will collect annotations from the sources listed in the `annotation-sources.txt` configuration file.
-The default file comes with sources of DNS-related annotations created by ICANN (such as errata and updates discussion)
-as well as outside sources that have told ICANN they want to be included.
+The directory already contains some directories initialized as git submodules.
+Be sure that the submodules are initialized with `git submodule update --init`.
+One submodule in the `annotations/from-icann` directory contains DNS-related annotations created by ICANN (such as errata and updates discussion).
+The other submodule directories come from outside sources that have told ICANN they want to be included.
 
 See the README in the `annotations/` folder for a description of the format of the annotation files.
 

program/drafts.py

@@ -2,10 +2,9 @@
 import os
 import subprocess
 from typing import Optional
-from urllib.request import urlopen
 from xml.dom.minidom import Document, parseString, Element
 
-import util  # filtered_files, debug, info, error
+import util  # filtered_files, debug, info, error, urlopen
 
 ''' Read and process Internet Drafts for RFC annotations tools '''
 
@@ -77,7 +76,7 @@ def get_draft_status(directory: str, url: str = "https://www.ietf.org/id/all_id.
     if document is None:
         util.info(f"\nFetching draft status from source of truth {url}... ", end='')
         try:
-            text_content = urlopen(url).read().decode('utf-8')
+            text_content = util.urlopen(url).read().decode('utf-8')
             util.info(f"Retrieved {len(text_content)} chars of data. Parsing and converting...", end='')
             document = {}
             for entry in text_content.split("\n"):

program/errata.py

@@ -1,9 +1,8 @@
 import json
 import os
 from typing import Optional
-from urllib.request import urlopen
 
-import util  # correct_path, create_checksum, config_directories, debug, info, error
+import util  # correct_path, create_checksum, config_directories, debug, info, error, urlopen
 
 ''' Create errata for RFC annotations tools '''
 
@@ -22,7 +21,7 @@ def read_errata(path: str = ".", url: str = "https://www.rfc-editor.org/errata.j
     if document is None:
         util.info(f"\nFetching errata from source of truth {url}... ", end='')
         try:
-            json_content = urlopen(url).read()
+            json_content = util.urlopen(url).read()
             util.info("Done")
             if type(json_content) is bytes:
                 util.debug(f"Retrieved {len(json_content)} bytes of data. Parsing... ", end='')

program/rfcfile.py

@@ -1,7 +1,6 @@
 import os
-from urllib.request import urlopen
 
-import util  # correct_path, debug, info, error
+import util  # correct_path, debug, info, error, urlopen
 
 ''' Download the RFC files for RFC annotations tools '''
 
@@ -24,7 +23,7 @@ def download_rfcs(rfc_list: list, directory: str = "."):
         else:
             util.info(f"Downloading {rfc.ljust(7)}... ", end='')
             try:
-                content = urlopen(f"https://www.rfc-editor.org/rfc/{rfc}.txt").read()
+                content = util.urlopen(f"https://www.rfc-editor.org/rfc/{rfc}.txt").read()
                 if type(content) is bytes:
                     util.info(f"Retrieved {str(len(content)).rjust(6)} bytes of data.")
                     with open(filename, "wb") as f:
@@ -34,4 +33,4 @@ def download_rfcs(rfc_list: list, directory: str = "."):
                     util.error(f"got unexpected fetching response data of type {type(content)}.")
             except Exception as e:
                 util.error(f"can't download text file for {rfc}: {e}.")
-    util.info(f"All RFC documents handled.")
+    util.info(f"All RFC documents handled.")

program/util.py

@@ -3,6 +3,7 @@
 import re
 import sys
 from typing import Optional
+import urllib.request
 
 ''' Utility functions for RFC annotations tools '''
 
@@ -193,3 +194,6 @@ def means_true(s: str) -> bool:
 
 def config_directories() -> [str]:
     return ["default-config"] if _running_in_test else ["local-config", "default-config"]
+
+def urlopen(url):
+	return urllib.request.urlopen(urllib.request.Request(url, headers ={'User-Agent': 'Mozilla/5.0'}))