huge tests WiP

.gitignore

@@ -32,7 +32,8 @@ dmypy.json
 # PyCharm
 .idea/
 
-# tests    poetry install
+# tests
 
+.nox/
 .tox/
 .coverage

many_repos/config.py

@@ -25,12 +25,16 @@ class Source(NamedTuple):
 
 class Config(NamedTuple):
     output_dir: str | Path
-    sources_configs: dict[str, Source] | None
+    sources_configs: dict[str, Source] | dict
 
     @classmethod
     def from_dict(cls, _dict: dict[str, Any]) -> Self:
-        output_dir = _dict.get("config").get("output_dir", None)
+
-        sources = {name: Source.from_dict(src) for name, src in _dict.get("sources").items()}
+        if not (cfg := _dict.get("config")):
+            raise errors.InvalidConfig("`[config]` cannot be empty!")
+
+        output_dir = cfg.get("output_dir", None)
+        sources = {name: Source.from_dict(src) for name, src in _dict.get("sources", dict()).items()}
         if not output_dir:
             raise errors.InvalidConfig("`output_dir` needs to be specified!")
         cfg = cls(output_dir=output_dir, sources_configs=sources)

many_repos/errors.py

@@ -11,3 +11,8 @@ class InvalidConfig(Exception):
 class InvalidSource(Exception):
     """Raised when invalid source is used"""
     pass
+
+
+class HTTPError(Exception):
+    """Raised when API rejects our request"""
+    pass

many_repos/source_type/__init__.py

@@ -1,13 +1,4 @@
-import importlib
+from .utils import get
-
+from . import github
-from . import base
+from . import gitlab
-from .. import errors
+from . import gitlab_group
-
-
-def get(source_name: str) -> type[base.BaseSource]:
-    try:
-        module = importlib.import_module(f"many_repos.source_type.{source_name}")
-        return module.Source
-    except (ModuleNotFoundError, AttributeError):
-        raise errors.InvalidSource(f"Source {source_name} not found!")
-

many_repos/source_type/base.py

@@ -1,8 +1,9 @@
 import abc
 import json
+from http.client import HTTPResponse
 from urllib import request
 
-from many_repos import common
+from many_repos import common, errors
 from many_repos import config
 
 
@@ -36,6 +37,9 @@ class BaseSource(abc.ABC):
         """
         Helper function to create http requests using urllib
         """
+        if not headers:
+            headers = dict()
+
         if body:
             headers["Content-Type"] = "application/json"
             data = json.dumps(body).encode('utf-8')
@@ -48,5 +52,12 @@ class BaseSource(abc.ABC):
             headers=headers,
             data=data
         )
-        res = request.urlopen(req)
+
+        res: HTTPResponse = request.urlopen(req)
+
+        if 400 <= res.status < 500:
+            raise errors.HTTPError(f"Client failed to make http request: {res.reason}")
+        if 500 <= res.status < 600:
+            raise errors.HTTPError(f"Server failed to respond to our request: {res.reason}")
+
         return json.load(res)

many_repos/source_type/github.py

@@ -11,13 +11,13 @@ class Source(BaseSource):
         for repo in repos_json:
             if not self.source_config.forks and repo.get("fork"):
                 continue
-            namespace, name = repo.get("full_name").split("/", 2)
+            namespace, name = repo["full_name"].split("/", 2)
             repositories.append(
                 common.Repository(
                     name=name,
                     namespace=namespace,
-                    url=repo.get("ssh_url"),
+                    url=repo["ssh_url"],
-                    fork=repo.get("fork"),
+                    fork=repo["fork"],
                     vcs="github.com"
                 )
             )

many_repos/source_type/gitlab.py

@@ -7,13 +7,13 @@ class Source(BaseSource):
         repos_json = self._make_request(self._api_url, headers=self._headers)
         repositories = []
         for repo in repos_json:
-            namespace, name = repo.get("path_with_namespace").split("/")
+            namespace, name = repo["path_with_namespace"].split("/")
             repositories.append(
                 common.Repository(
                     name=name,
                     namespace=namespace,
-                    url=repo.get("ssh_url_to_repo"),
+                    url=repo["ssh_url_to_repo"],
-                    fork=bool(repo.get("forked_from_project")),
+                    fork=bool(repo["forked_from_project"]),
                     vcs="gitlab.com"
                 )
             )

many_repos/source_type/utils.py

@@ -0,0 +1,12 @@
+import importlib
+
+from . import base
+from .. import errors
+
+
+def get(source_name: str) -> type[base.BaseSource]:
+    try:
+        module = importlib.import_module(f"many_repos.source_type.{source_name}")
+        return module.Source
+    except (ModuleNotFoundError, AttributeError):
+        raise errors.InvalidSource(f"Source {source_name} not found!")

noxfile.py

@@ -9,11 +9,11 @@ def tests(s):
     s.run('pytest', 'tests')
 
 
-@session(python='python3.12')
+@session(python='python3.12', name="type")
 def _type(s):
     """run type checks"""
     s.install('mypy')
-    s.run('mypy', 'many_repos', 'tests')
+    s.run('mypy', 'many_repos')
 
 
 @session(python='python3.12')

pyproject.toml

@@ -16,7 +16,6 @@ many-repos-clone = "many_repos.clone:main"
 [tool.poetry.dependencies]
 python = "^3.11"
 
-
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.1"
 ruff = "^0.4.5"

tests/common_test.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+from many_repos.clone import _construct_path
+
+
+def test_correct_construct_path(test_repo, many_repos_config):
+    assert _construct_path(test_repo, many_repos_config) == (
+                Path(many_repos_config.output_dir) / test_repo.vcs / test_repo.namespace / test_repo.name).resolve()

tests/conftest.py

@@ -0,0 +1,67 @@
+import pytest
+
+from many_repos.common import Repository
+from many_repos.config import Source, Config
+
+
+@pytest.fixture(params=[True, False])
+def source_config(request):
+    return Source(
+        source_type="github",
+        username="JustScreaMy",
+        token="testToken",
+        forks=request.param
+    )
+
+
+@pytest.fixture
+def test_repo():
+    return Repository(
+        name="many-repos",
+        namespace="JustScreaMy",
+        url="git@github.com/JustScreaMy/many-repos.git",
+        fork=False,
+        vcs="github.com"
+    )
+
+
+@pytest.fixture
+def many_repos_config():
+    return Config(
+        output_dir="/tmp/test/repos",
+        sources_configs={
+            "gitlab": Source(
+                source_type="gitlab",
+                username="JustScreaMy",
+                token="testToken",
+                forks=False
+            ),
+            "github": Source(
+                source_type="github",
+                username="JustScreaMy",
+                token="testTokasden",
+                forks=False
+            )
+        }
+    )
+
+
+@pytest.fixture
+def invalid_many_repos_config():
+    return Config(
+        output_dir="/tmp/test/repos",
+        sources_configs={
+            "gitlab": Source(
+                source_type="gitlab",
+                username="JustScreaMy",
+                token="testToken",
+                forks=False
+            ),
+            "bitbucket": Source(
+                source_type="bitbucket",
+                username="JustScreaMy",
+                token="testTokasden",
+                forks=False
+            )
+        }
+    )

tests/dummy_test.py

@@ -1,3 +0,0 @@
-class TestDummy:
-    def test_dummy(self):
-        assert 1 == 1

tests/source_type/conftest.py

@@ -1,13 +0,0 @@
-import pytest
-
-from many_repos.config import Source
-
-
-@pytest.fixture(params=[True, False])
-def source_config(request):
-    return Source(
-        source_type="github",
-        username="JustScreaMy",
-        token="testToken",
-        forks=request.param
-    )

tests/source_type/github_test.py

@@ -1,7 +1,5 @@
 from many_repos.source_type.github import Source as GithubSource
-from unittest import mock
 
 
-class TestGithubSource:
+def dummy():
-    def test_api_url_generation(self, source_config, monkeypatch):
+    assert 1 == 1
-        src = GithubSource(source_config)

tests/source_type/utils_test.py

@@ -0,0 +1,21 @@
+import pytest
+
+from many_repos import errors
+from many_repos import source_type
+
+
+@pytest.mark.parametrize(
+    "source, source_model",
+    (
+            ("gitlab", source_type.gitlab.Source),
+            ("gitlab_group", source_type.gitlab_group.Source),
+            ("github", source_type.github.Source)
+    )
+)
+def test_correct_source_model(source, source_model):
+    assert source_type.get(source) == source_model
+
+
+def test_invalid_source_model():
+    with pytest.raises(errors.InvalidSource):
+        source_type.get("bitbucket")