Merge pull request #40 from jadchaar/feature/def-14a

Added support for DEF 14A and fixed a few bugs; reorganized tests

.github/workflows/continuous_integration.yml

@@ -22,7 +22,7 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pip install -U pip setuptools wheel
           pip install tox
       - name: Linting code
         run: tox -e lint
@@ -43,7 +43,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pip install -U pip setuptools wheel
           pip install tox tox-gh-actions
       - name: Test with tox
         run: tox
@@ -67,7 +67,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pip install -U pip setuptools wheel
           pip install tox tox-gh-actions
       - name: Test with tox
         run: tox
@@ -91,7 +91,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pip install -U pip setuptools wheel
           pip install tox tox-gh-actions
       - name: Test with tox
         run: tox

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: bandit
         args: [-lll, --quiet, --exclude=tests/**]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v3.0.0
     hooks:
       - id: check-case-conflict
       - id: check-docstring-first
@@ -30,7 +30,7 @@ repos:
       - id: isort
         args: [--quiet]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.2.0
+    rev: v2.4.1
     hooks:
       - id: pyupgrade
         args: [--py36-plus]
@@ -45,7 +45,7 @@ repos:
       - id: black
         args: [--safe, --quiet, --target-version=py36]
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.8.1
     hooks:
       - id: flake8
         additional_dependencies: [flake8-bugbear]

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 3.0.5 - 5/16/2020
+
+### Added
+
+- Added support for DEF 14A filings (proxy statements).
+- Added `tox.ini` and `Makefile` to distribution package.
+
+### Fixed
+
+- Fixed a failing test in the distributed package due to missing sample filings test data.
+
 ## 3.0.4 - 4/21/2020
 
 ### Added

MANIFEST.in

@@ -1,3 +1,3 @@
-include LICENSE CHANGELOG.md README.rst requirements.txt
+include LICENSE CHANGELOG.md README.rst Makefile requirements.txt tox.ini
 recursive-include tests *.py
 recursive-include docs *.py *.rst *.bat Makefile

Makefile

@@ -1,32 +1,42 @@
-.PHONY: test clean docs publish
+.PHONY: auto test docs clean
+
+auto: build38
 
 build36: PYTHON_VER = python3.6
 build37: PYTHON_VER = python3.7
 build38: PYTHON_VER = python3.8
 
-build36 build37 build38:
-	virtualenv venv --python=$(PYTHON_VER)
-	venv/bin/pip install -r requirements.txt
-	venv/bin/pre-commit install
+build36 build37 build38: clean
+	$(PYTHON_VER) -m venv venv
+	. venv/bin/activate; \
+	pip install -U pip setuptools wheel; \
+	pip install -r requirements.txt; \
+	pre-commit install
 
 test:
 	rm -f .coverage coverage.xml
-	. venv/bin/activate && pytest
+	. venv/bin/activate; pytest
 
 lint:
-	venv/bin/pre-commit run --all-files --show-diff-on-failure
-
-clean:
-	rm -rf venv .tox .pytest_cache ./**/__pycache__
-	rm -rf dist build .egg .eggs sec_edgar_downloader.egg-info
-	rm -f ./**/*.pyc .coverage coverage.xml
+	. venv/bin/activate; pre-commit run --all-files --show-diff-on-failure
 
 docs:
-	cd docs; make html
+	rm -rf docs/_build
+	. venv/bin/activate; cd docs; make html
 
-publish:
-	rm -rf dist build .egg .eggs sec_edgar_downloader.egg-info
-	pip3 install -U setuptools twine wheel
-	python3 setup.py sdist bdist_wheel
-	twine upload dist/*
+clean: clean-dist
+	rm -rf venv .pytest_cache ./**/__pycache__
+	rm -f .coverage coverage.xml ./**/*.pyc
+
+clean-dist:
 	rm -rf dist build .egg .eggs sec_edgar_downloader.egg-info
+
+build-dist:
+	. venv/bin/activate; \
+	pip install -U setuptools twine wheel; \
+	python setup.py sdist bdist_wheel
+
+upload-dist:
+	. venv/bin/activate; twine upload dist/*
+
+publish: test clean-dist build-dist upload-dist clean-dist

README.rst

@@ -112,6 +112,7 @@ Supported SEC Filings
 - SC 13G
 - SD
 - S-1
+- DEF 14A
 
 You can learn more about the different types of SEC filings `here <https://www.investopedia.com/articles/fundamental-analysis/08/sec-forms.asp>`_.
 

requirements.txt

@@ -2,4 +2,4 @@ lxml>=4.3.4
 pre-commit
 pytest
 pytest-cov
-requests>=2.22.0
+requests

sec_edgar_downloader/_constants.py

@@ -13,5 +13,6 @@
     "SC 13G",
     "SD",
     "S-1",
+    "DEF 14A",
 }
 W3_NAMESPACE = {"w3": "http://www.w3.org/2005/Atom"}

sec_edgar_downloader/_version.py

@@ -1 +1 @@
-__version__ = "3.0.4"
+__version__ = "3.0.5"

setup.py

@@ -19,7 +19,7 @@
     url="https://github.com/jadchaar/sec-edgar-downloader",
     packages=["sec_edgar_downloader"],
     zip_safe=False,
-    install_requires=["lxml>=4.3.4", "requests>=2.22.0"],
+    install_requires=["lxml>=4.3.4", "requests"],
     python_requires=">=3.6",
     classifiers=[
         "Development Status :: 5 - Production/Stable",

tests/test_filing_downloads.py → tests/test_file_structure_and_content.py

@@ -79,4 +79,8 @@ def test_file_contents(downloader):
 
     # https://stackoverflow.com/q/1072569
     expected_data_path = Path(f"tests/sample_filings/apple_8k_{before_date}.txt")
-    assert filecmp.cmp(expected_data_path, downloaded_file_path, shallow=False)
+    if expected_data_path.exists():
+        # Only run this check if the sample filing exists
+        # This check is required since the distributed python package will not
+        # contain the sample filings test data due to size constraints
+        assert filecmp.cmp(expected_data_path, downloaded_file_path, shallow=False)

tests/test_filing_url_retrieval.py

@@ -1,9 +1,11 @@
 """Tests the filing URLs to download for each filing type."""
 
+import pytest
+
 from sec_edgar_downloader._utils import get_filing_urls_to_download
 
 
-def test_filing_url_retrieval():
+def test_large_number_of_filings():
     filing_type = "8-K"
     ticker = "AAPL"
     after_date = None
@@ -62,49 +64,46 @@ def test_filing_url_retrieval():
     assert len(filings_to_download) == 176
 
 
-def test_common_filings():
-    # AAPL files 4, 8-K, 10-K, 10-Q, SC 13G, SD
+@pytest.mark.parametrize(
+    "filing_type", ["4", "8-K", "10-K", "10-Q", "SC 13G", "SD", "DEF 14A"]
+)
+def test_common_filings(filing_type):
+    # AAPL files 4, 8-K, 10-K, 10-Q, SC 13G, SD, DEF 14A
     ticker = "AAPL"
-    filing_types = ["4", "8-K", "10-K", "10-Q", "SC 13G", "SD"]
     num_filings_to_download = 1
     after_date = None
     before_date = None
     include_amends = False
 
-    for filing_type in filing_types:
-        filings_to_download = get_filing_urls_to_download(
-            filing_type,
-            ticker,
-            num_filings_to_download,
-            after_date,
-            before_date,
-            include_amends,
-        )
-        assert len(filings_to_download) == 1
+    filings_to_download = get_filing_urls_to_download(
+        filing_type,
+        ticker,
+        num_filings_to_download,
+        after_date,
+        before_date,
+        include_amends,
+    )
+    assert len(filings_to_download) == 1
 
 
-def test_13f_filings():
+@pytest.mark.parametrize("filing_type", ["13F-NT", "13F-HR"])
+def test_13f_filings(filing_type):
     # Vanguard files 13F-NT, 13F-HR
     ticker = "0000102909"
-    filing_types = [
-        "13F-NT",
-        "13F-HR",
-    ]
     num_filings_to_download = 1
     after_date = None
     before_date = None
     include_amends = False
 
-    for filing_type in filing_types:
-        filings_to_download = get_filing_urls_to_download(
-            filing_type,
-            ticker,
-            num_filings_to_download,
-            after_date,
-            before_date,
-            include_amends,
-        )
-        assert len(filings_to_download) == 1
+    filings_to_download = get_filing_urls_to_download(
+        filing_type,
+        ticker,
+        num_filings_to_download,
+        after_date,
+        before_date,
+        include_amends,
+    )
+    assert len(filings_to_download) == 1
 
 
 def test_10ksb_filings():

tox.ini

@@ -8,6 +8,8 @@ python =
     3.7: py37
     3.8: py38
 
+# Configure testing environments
+
 [testenv]
 description = run the tests with pytest using {basepython}
 deps = -rrequirements.txt
@@ -36,24 +38,19 @@ commands =
     doc8 index.rst ../README.rst --extension .rst --ignore D001
     make html SPHINXOPTS="-W --keep-going"
 
+# Configure settings for pytest, isort, and flake8
+
 [pytest]
 addopts = -v --cov-branch --cov=sec_edgar_downloader --cov-fail-under=100 --cov-report=term-missing --cov-report=xml
 
+[isort]
+line_length = 88
+multi_line_output = 3
+include_trailing_comma = true
+known_third_party = lxml,pytest,requests,setuptools
+
 [flake8]
 max-line-length = 88
 select = C,E,F,W,B,B9
 per-file-ignores = sec_edgar_downloader/__init__.py:F401
 ignore = E203,E501,W503
-exclude =
-    .git,
-    __pycache__,
-    *.pyc,
-    docs/conf.py,
-    build,
-    dist
-
-[tool:isort]
-line_length = 88
-multi_line_output = 3
-include_trailing_comma = true
-known_third_party = lxml,pytest,requests,setuptools