From 4de08dd2390b6195da7ac99723931f86ef270b50 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 08:38:18 +0200 Subject: [PATCH 01/16] docs: fixes pip install command --- .gitlab-ci.yml | 1 + readme.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b892782..84935ee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ include: variables: PYTHON_PACKAGE: dgad + GITLAB_RECOMMENDED_AUTO_FIX: "true" dgad-cli: stage: test diff --git a/readme.md b/readme.md index 69b5de7..744bdcf 100644 --- a/readme.md +++ b/readme.md @@ -14,7 +14,7 @@ # recommended: use a virtual environment python -m venv dgad source dgad/bin/activate -add pip install git+https://gitlab.com/cossas/dgad.git +pip install git+https://gitlab.com/cossas/dgad.git ``` ## CLI -- GitLab From 254b63249ca6175cd9dfdb11ab6d87369174cce4 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 09:10:56 +0200 Subject: [PATCH 02/16] fix: changes base image to python slim --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1ab8f48..2b2e21b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -FROM ubuntu:latest -RUN apt -y update && apt install -y python3 python3-pip +# FROM ubuntu:latest +# RUN apt -y update && apt install -y python3 python3-pip +FROM python:3.9-slim WORKDIR /project COPY pyproject.toml . COPY dgad dgad -- GitLab From b6fa6ffef45f9c2c17a901d57080ea614223b989 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 09:17:54 +0200 Subject: [PATCH 03/16] test --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2b2e21b..19eeec2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM python:3.9-slim WORKDIR /project COPY pyproject.toml . COPY dgad dgad -RUN pip install . +RUN pip install redis COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 -- GitLab From dfa8976ccae8e40dd64dc5c991f9d29effdb3f12 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 09:22:32 +0200 Subject: [PATCH 04/16] fix: try with tensorflow image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 19eeec2..601bfee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # FROM ubuntu:latest # RUN apt -y update && apt install -y python3 python3-pip -FROM python:3.9-slim +FROM tensorflow/tensorflow:2.6.0 WORKDIR /project COPY pyproject.toml . COPY dgad dgad -RUN pip install redis +RUN pip install . COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 -- GitLab From 6076a8a08b4b6f0268b06d735fa09b4c44a110d5 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 09:49:13 +0200 Subject: [PATCH 05/16] try again with pyton-slim --- Dockerfile | 4 +--- poetry.lock | 45 ++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 +- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 601bfee..ddcb2e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ -# FROM ubuntu:latest -# RUN apt -y update && apt install -y python3 python3-pip -FROM tensorflow/tensorflow:2.6.0 +FROM python:3.9-slim WORKDIR /project COPY pyproject.toml . COPY dgad dgad diff --git a/poetry.lock b/poetry.lock index 44ccb64..76d9903 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1460,6 +1460,35 @@ tensorflow = ["tensorflow (>=2.4.0,<2.7.0)"] tensorflow-cpu = ["tensorflow-cpu (>=2.4.0,<2.7.0)"] tensorflow-gpu = ["tensorflow-gpu (>=2.4.0,<2.7.0)"] +[[package]] +name = "tensorflow-cpu" +version = "2.6.0" +description = "TensorFlow is an open source machine learning framework for everyone." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +absl-py = ">=0.10,<1.0" +astunparse = ">=1.6.3,<1.7.0" +clang = ">=5.0,<6.0" +flatbuffers = ">=1.12.0,<1.13.0" +gast = "0.4.0" +google-pasta = ">=0.2,<1.0" +grpcio = ">=1.37.0,<2.0" +h5py = ">=3.1.0,<3.2.0" +keras = ">=2.6,<3.0" +keras-preprocessing = ">=1.1.2,<1.2.0" +numpy = ">=1.19.2,<1.20.0" +opt-einsum = ">=3.3.0,<3.4.0" +protobuf = ">=3.9.2" +six = ">=1.15.0,<1.16.0" +tensorboard = ">=2.6,<3.0" +tensorflow-estimator = ">=2.6,<3.0" +termcolor = ">=1.1.0,<1.2.0" +typing-extensions = ">=3.7.4,<3.8.0" +wrapt = ">=1.12.1,<1.13.0" + [[package]] name = "tensorflow-estimator" version = "2.6.0" @@ -1710,7 +1739,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6.1" -content-hash = "90d0f783a12d2d853c2d48bea94a68c5dbe912339d35afa6b46698ca60b4be73" +content-hash = "dd6c04933693e15f74b245464d1919f6d66b9d974e9e00bcb4a4d198bf02fb75" [metadata.files] absl-py = [ @@ -2631,6 +2660,20 @@ tensorflow-addons = [ {file = "tensorflow_addons-0.14.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a4f9bde8854b8f4bd2d74e00e6e8d7233a85372d9d1d2aad1d839e232f3d257d"}, {file = "tensorflow_addons-0.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:1aaabb662b718caaae4fc5014e7d5b81fa7a9ab9328385bc0b3838b88360fc12"}, ] +tensorflow-cpu = [ + {file = "tensorflow_cpu-2.6.0-cp36-cp36m-macosx_10_11_x86_64.whl", hash = "sha256:060f6c6b1b3299be0288179f4a5c49f29a332feb1a023d8840246ccd1085a1df"}, + {file = "tensorflow_cpu-2.6.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:74e27b3fa369d39130f25690b45fd792f1b6e771ffa80419677165f6b9b53795"}, + {file = "tensorflow_cpu-2.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d95ef3e9fa34dbee9b9903f52c9e2811e7ad17fd5f4ea108d8b3a5a5ccf3ca04"}, + {file = "tensorflow_cpu-2.6.0-cp37-cp37m-macosx_10_11_x86_64.whl", hash = "sha256:39d8ecf65ab3f9738be740268cb9645b3fcdf08fd42b591873dfb4d1a953c8ad"}, + {file = "tensorflow_cpu-2.6.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:49572a019358fb0b6ea6a36bcb39d179278e3a88724ae23c7bccc3291d3f0bec"}, + {file = "tensorflow_cpu-2.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4790031201f3c3f626f8218600680353bbed6797353c897b5e59332c804f7e93"}, + {file = "tensorflow_cpu-2.6.0-cp38-cp38-macosx_10_11_x86_64.whl", hash = "sha256:f88bbedb2b8b67303c80240e76cb72f831e3659c1ebe9276fe530f26ec86929d"}, + {file = "tensorflow_cpu-2.6.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:8322106d9c4d48f8378f6a37aee231415b58c25a7223d96c6e5f1e37f9b129d8"}, + {file = "tensorflow_cpu-2.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:503eea2b7c3d87d305d4539a9d93115550ba22f4828cc00dee76ecb4057c7c63"}, + {file = "tensorflow_cpu-2.6.0-cp39-cp39-macosx_10_11_x86_64.whl", hash = "sha256:616fbbe473bb8df58a4060dd062582bec36ae4bb1f516e178ffeed5f92f43cdf"}, + {file = "tensorflow_cpu-2.6.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:d2458b6e061151048d89fabf82207eb8a7e0527b64bdd6ac61c5a5d5255d087e"}, + {file = "tensorflow_cpu-2.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:c65bf8665d26933038eaa1e99f525ff7832f1d99832f1ea9ca2068ef23715b2c"}, +] tensorflow-estimator = [ {file = "tensorflow_estimator-2.6.0-py2.py3-none-any.whl", hash = "sha256:cf78528998efdb637ac0abaf525c929bf192767544eb24ae20d9266effcf5afd"}, ] diff --git a/pyproject.toml b/pyproject.toml index ab05a1a..c2a22c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,11 +11,11 @@ python = "^3.6.1" pandas = "^1.1.5" tldextract = "^3.1.0" redis = "^3.5.3" -tensorflow = "^2.4.0" numpy = "^1.19.5" keras-tcn = "^3.2.1" grpcio = "^1.34" pip = "^21.1.2" +tensorflow-cpu = "^2.6.0" [tool.poetry.dev-dependencies] black = "^20.8b1" -- GitLab From 15449e838d12ff206cf9d5c41a17e5073bb83005 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 09:59:31 +0200 Subject: [PATCH 06/16] chore: updates safe licenses --- pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c2a22c3..3d306c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,5 +52,14 @@ build_command = "pip install poetry && poetry build" safe_licenses = [ "Apache Software License", "MIT License", - "BSD License" + "BSD License", + "Apache 2.0", + "MIT", + "Python Software Foundation License", + "3-Clause BSD License", + "Public Domain", + "BSD", + "Mozilla Public License 2.0 (MPL 2.0)", + "University of Illinois/NCSA Open Source License", + "GNU Lesser General Public License v3 or later (LGPLv3+)" ] -- GitLab From 9029db1d3c104af2c76862cb2de4dfb4f3d7b824 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 10:14:46 +0200 Subject: [PATCH 07/16] fix: fixes to urls and tbump --- skaffold.yaml | 2 +- tbump.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/skaffold.yaml b/skaffold.yaml index 8d35e8b..7fe77d4 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -4,7 +4,7 @@ metadata: name: dgad build: artifacts: - - image: + - image: registry.gitlab.com/cossas/dgad:v3.0.4 docker: dockerfile: Dockerfile deploy: diff --git a/tbump.toml b/tbump.toml index 1c20257..ddc6d56 100644 --- a/tbump.toml +++ b/tbump.toml @@ -28,6 +28,8 @@ src = "readme.md" src = "pyproject.toml" [[file]] src = "skaffold/deployment.yaml" +[[file]] +src = "skaffold.yaml" # You can specify a list of commands to # run after the files have been patched -- GitLab From 05c9051b16109b3a94e489c625b237db244f2eb5 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 10:38:32 +0200 Subject: [PATCH 08/16] centos image --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ddcb2e5..597ea2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ -FROM python:3.9-slim +FROM centos:8 +RUN yum -y update && yum -y install python3.9 python3-pip && yum clean all && rm -rf /var/cache/yum WORKDIR /project COPY pyproject.toml . COPY dgad dgad -RUN pip install . +RUN pip3 install . COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 -- GitLab From 87f78408b82889b31eeebcc411ead27e97d5fc53 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 10:49:23 +0200 Subject: [PATCH 09/16] build: try multistage to slim down image --- Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 597ea2f..8dfe061 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,13 @@ -FROM centos:8 -RUN yum -y update && yum -y install python3.9 python3-pip && yum clean all && rm -rf /var/cache/yum +FROM python:3.9-slim as builder WORKDIR /project COPY pyproject.toml . COPY dgad dgad -RUN pip3 install . +RUN pip install --prefix=/install . + +FROM python:3.9-alpine +COPY --from=builder /install /usr/local +WORKDIR /project +COPY dgad dgad COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 -- GitLab From 51e50e56f36bb7ee2f430d0680faa0b9343f8394 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 10:55:12 +0200 Subject: [PATCH 10/16] build: try multistage and centos --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8dfe061..4982f8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ COPY pyproject.toml . COPY dgad dgad RUN pip install --prefix=/install . -FROM python:3.9-alpine +FROM centos:8 +RUN yum -y update && yum -y install python3.9 && yum clean all && rm -rf /var/cache/yum COPY --from=builder /install /usr/local WORKDIR /project COPY dgad dgad -- GitLab From 26ee2c65924305d002e4c333cfcd7492f680d5fd Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 11:06:03 +0200 Subject: [PATCH 11/16] fix: adds pip to image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4982f8a..1831403 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ COPY dgad dgad RUN pip install --prefix=/install . FROM centos:8 -RUN yum -y update && yum -y install python3.9 && yum clean all && rm -rf /var/cache/yum +RUN yum -y update && yum -y install python3.9 python3-pip && yum clean all && rm -rf /var/cache/yum COPY --from=builder /install /usr/local WORKDIR /project COPY dgad dgad -- GitLab From b2a32fbe4b3b48dadb3ab430d21addb693fbad20 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 12:41:21 +0200 Subject: [PATCH 12/16] fix: removes python3-pip, which installs python3.6 pip3 should be already present wiht python3 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1831403..4982f8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ COPY dgad dgad RUN pip install --prefix=/install . FROM centos:8 -RUN yum -y update && yum -y install python3.9 python3-pip && yum clean all && rm -rf /var/cache/yum +RUN yum -y update && yum -y install python3.9 && yum clean all && rm -rf /var/cache/yum COPY --from=builder /install /usr/local WORKDIR /project COPY dgad dgad -- GitLab From dae7aa2bef59aad0fb2a607872f34243f88905d3 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 13:22:17 +0200 Subject: [PATCH 13/16] fix: adds alias to point python to right executable --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 4982f8a..95443c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,6 @@ COPY dgad dgad COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 +RUN alias python='/usr/bin/python3.9' ENTRYPOINT [ "dgad"] CMD [ "-h" ] -- GitLab From 901b598886d48feee9d45c4eb4256ee79f1cac8e Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 13:50:47 +0200 Subject: [PATCH 14/16] ci: removes redundant allow_failure --- .gitlab-ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 84935ee..8288f46 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,12 +14,3 @@ dgad-cli: - dgad --domain wikipedia.org - dgad --domains wikipedia.org sjdkahflaksdjhf.net - dgad --csv tests/data/domains_todo.csv - -python:safety: - allow_failure: true - -python:bandit: - allow_failure: true - -grype: - allow_failure: true -- GitLab From be2420af952d6d71d1fe99a3f4771e76548ecae3 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 13:53:27 +0200 Subject: [PATCH 15/16] fix: try with symlink --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 95443c2..2f799e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ COPY dgad dgad COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 -RUN alias python='/usr/bin/python3.9' +RUN ln -s /usr/bin/python3 /usr/bin/python & \ + ln -s /usr/bin/pip3 /usr/bin/pip ENTRYPOINT [ "dgad"] CMD [ "-h" ] -- GitLab From 2ef835a5b2810743072927b7fe3303ae72d3f7d5 Mon Sep 17 00:00:00 2001 From: Federico Falconieri Date: Tue, 14 Sep 2021 14:02:47 +0200 Subject: [PATCH 16/16] fix: last attempt --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2f799e0..5313f1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY dgad dgad COPY tests/*.py tests/ COPY tests/data tests/data ENV TF_CPP_MIN_LOG_LEVEL=3 -RUN ln -s /usr/bin/python3 /usr/bin/python & \ +RUN ln -s /usr/bin/python3.9 /usr/bin/python & \ ln -s /usr/bin/pip3 /usr/bin/pip ENTRYPOINT [ "dgad"] CMD [ "-h" ] -- GitLab