From 5fb0fe6ee7250e584860fc5ca81ad89a3ce34504 Mon Sep 17 00:00:00 2001 From: Naramsim Date: Wed, 30 Jan 2019 17:14:09 +0100 Subject: [PATCH 1/3] feat: avoid redirects --- pokeapi_ditto/commands/clone.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pokeapi_ditto/commands/clone.py b/pokeapi_ditto/commands/clone.py index f4d01bba3..3ed7dee8a 100644 --- a/pokeapi_ditto/commands/clone.py +++ b/pokeapi_ditto/commands/clone.py @@ -11,7 +11,7 @@ def _do_in_parallel(worker: Callable, data: List, desc: str) -> None: - cpus = os.cpu_count() + cpus = os.cpu_count() - 1 pool = Pool(cpus, initializer=lambda: signal(SIGINT, SIG_IGN)) try: for _ in tqdm(pool.imap_unordered(worker, data), total=len(data), desc=f"{desc} ({cpus}x)"): @@ -65,10 +65,10 @@ def _crawl_resource_list(self, url: URL) -> List[URL]: def clone_single(self, endpoint_and_id: Tuple[str, str]) -> None: endpoint, id = endpoint_and_id - res_url = self._src_url / "api/v2" / endpoint / id + res_url = URL("{}/api/v2/{}/{}/".format(self._src_url, endpoint, id)) self._crawl(res_url) if endpoint == "pokemon": - self._crawl(res_url / "encounters") + self._crawl(URL("{}encounters/".format(res_url))) def clone_endpoint(self, endpoint: str): res_list_url = self._src_url / "api/v2" / endpoint From 688849a989f555bcc1509f61b95ff5363787fe03 Mon Sep 17 00:00:00 2001 From: Naramsim Date: Wed, 30 Jan 2019 17:14:34 +0100 Subject: [PATCH 2/3] feat: add direct Docker support --- .dockerignore | 4 ++++ .gitignore | 3 +++ Dockerfile | 16 ++++++++++++++++ docker-compose.yml | 9 +++++++++ 4 files changed, 32 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..d0e2135ec --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +dist +data +_gen \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9b64b6d48..8d6becb81 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ __pycache__ pyvenv.cfg .venv pip-selfcheck.json + +/data +/_gen \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f6022d17c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3-alpine + +RUN pip install poetry + +ADD . /ditto + +WORKDIR /ditto + +RUN poetry install +CMD poetry run ditto clone --dest-dir ./data && \ + poetry run ditto clone --src-url http://localhost/ --dest-dir ./data --select pokemon/129 && \ + poetry run ditto analyze --data-dir ./data && \ + poetry run ditto transform \ + --base-url='https://pokeapi.co' \ + --src-dir=./data \ + --dest-dir=./_gen \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..f301be9be --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +version: "2.3" + +services: + ditto: + build: . + volumes: + - ./data:/ditto/data + - ./_gen:/ditto/_gen + network_mode: "host" From a156eba24beb5cdb002305e464ac09111c47b1df Mon Sep 17 00:00:00 2001 From: Naramsim Date: Wed, 30 Jan 2019 17:16:39 +0100 Subject: [PATCH 3/3] docs: update readme --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 6612ea770..3647c7d75 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,11 @@ This repository contains: pip install pokeapi-ditto ditto --help ``` + +## Docker + +You should have a Pokeapi server running on `localhost:80`. + +```sh +docker-compose up --build +```