From fa9a72027003a69ca76318d0c3aef9ea4da64031 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 09:52:13 +0200 Subject: [PATCH 01/35] Use custom image and build docker only when tagging --- build/ci/.gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index 22d0286..2e43a28 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: sjdweb/go-docker-build:1.10 +image: welance/build-golang:1.10 stages: - test @@ -27,9 +27,13 @@ unit_tests: build: stage: build script: - - make docker + - make docker TAG=$CI_COMMIT_TAG + only: + - tag docker-build: stage: publish script: - - make docker-push \ No newline at end of file + - make docker-push TAG=$CI_COMMIT_TAG + only: + - tag \ No newline at end of file -- GitLab From 4b6611198510fa6aff8cda9732dea9d6645b9a52 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 09:52:52 +0200 Subject: [PATCH 02/35] Enable coverage and tagging --- .gitignore | 1 + Makefile | 3 ++- README.md | 10 ++++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 5b3e348..60f9ed2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ vendor distill.db third_party/swagger dist +.testCoverage.txt diff --git a/Makefile b/Makefile index 307ea3f..f345649 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ GIT_DESCR = $(shell git describe) OUTPUTFOLDER = dist # docker image DOCKERIMAGE = welance/distill +TAG = $(shell git describe) # build paramters OS = linux ARCH = amd64 @@ -30,7 +31,7 @@ build-dist: $(GOFILES) test: test-all test-all: - @go test -v $(GOPACKAGES) + @go test -v $(GOPACKAGES) -coverprofile .testCoverage.txt bench: bench-all diff --git a/README.md b/README.md index d1182b0..05d6fcc 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ Another url shortener +[![pipeline status](https://gitlab.com/welance/distill/badges/develop/pipeline.svg)](https://gitlab.com/welance/distill/commits/develop) + +[![coverage report](https://gitlab.com/welance/distill/badges/develop/coverage.svg)](https://gitlab.com/welance/distill/commits/develop) + ## Motivations Existing url shorteners are not suitable for a private deploy use or are too complex in terms of requirements. @@ -97,5 +101,7 @@ test ## Development -to generate the Colfer model run -`colf -b internal Go api/model.colf` from the project root \ No newline at end of file +- to generate the Colfer model run +`colf -b internal Go api/model.colf` from the project root + +- to enable coverage badge use `^coverage:\s(\d+(?:\.\d+)?%)` as regexp in gilab configuration \ No newline at end of file -- GitLab From f2457105c2115d4cdddb3638459452ae85a75a84 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 12:38:12 +0200 Subject: [PATCH 03/35] Fix "only" filter for ci --- build/ci/.gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index 2e43a28..6b4f1e4 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -29,11 +29,11 @@ build: script: - make docker TAG=$CI_COMMIT_TAG only: - - tag + - tags docker-build: stage: publish script: - make docker-push TAG=$CI_COMMIT_TAG only: - - tag \ No newline at end of file + - tags \ No newline at end of file -- GitLab From 82700d8f9a63149aabe3c19d22183ab0aa832878 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 13:45:26 +0200 Subject: [PATCH 04/35] Add TAG when building Start docker before build --- Makefile | 4 ++-- build/ci/.gitlab-ci.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f345649..fe10c2b 100644 --- a/Makefile +++ b/Makefile @@ -54,12 +54,12 @@ docker-build: build-dist @echo copy resources @cp configs/settings.docker.yaml $(OUTPUTFOLDER)/settings.yaml @echo build image - docker build -t $(DOCKERIMAGE) -f ./build/docker/Dockerfile . + docker build -t $(DOCKERIMAGE):$(TAG) -f ./build/docker/Dockerfile . @echo done docker-push: docker-build @echo push image - docker push $(DOCKERIMAGE) + docker push $(DOCKERIMAGE):$(TAG) @echo done docker-run: diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index 6b4f1e4..b50b611 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -13,6 +13,7 @@ before_script: - cd $CI_PROJECT_NAME - echo -e "machine gitlab.com\nlogin gitlab-ci-token\npassword ${CI_JOB_TOKEN}" > ~/.netrc - dep ensure -vendor-only + - service docker start lint_code: stage: test -- GitLab From 58aa4abc63cd6ea651dece6599cc6495cfb03140 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 14:03:26 +0200 Subject: [PATCH 05/35] Add build for all the linux/mac/windows --- build/ci/.gitlab-ci.yml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index b50b611..cd0597e 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -25,14 +25,39 @@ unit_tests: script: - make test -build: +build-linux: + stage: build + script: + - make build OS=linux + artifacts: + paths: + - dist + +build-darwin: + stage: build + script: + - make build OS=darwin + artifacts: + paths: + - dist + +build-windows: + stage: build + script: + - make build OS=windows + artifacts: + paths: + - dist + + +docker-build: stage: build script: - make docker TAG=$CI_COMMIT_TAG only: - tags -docker-build: +docker-push: stage: publish script: - make docker-push TAG=$CI_COMMIT_TAG -- GitLab From c0943ab18fbfd97aa4d550161466dd997918bdf1 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 14:27:10 +0200 Subject: [PATCH 06/35] Disable docker builds --- build/ci/.gitlab-ci.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index cd0597e..af73fa9 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -13,7 +13,6 @@ before_script: - cd $CI_PROJECT_NAME - echo -e "machine gitlab.com\nlogin gitlab-ci-token\npassword ${CI_JOB_TOKEN}" > ~/.netrc - dep ensure -vendor-only - - service docker start lint_code: stage: test @@ -49,17 +48,3 @@ build-windows: paths: - dist - -docker-build: - stage: build - script: - - make docker TAG=$CI_COMMIT_TAG - only: - - tags - -docker-push: - stage: publish - script: - - make docker-push TAG=$CI_COMMIT_TAG - only: - - tags \ No newline at end of file -- GitLab From ca3611d80ab8e8f5b154fec40b8d7b6a46515b61 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 21:45:06 +0200 Subject: [PATCH 07/35] Add docker dind build and push --- build/ci/.gitlab-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index af73fa9..dcfa9e3 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -5,7 +5,11 @@ stages: - build - publish +services: + - docker:dind + before_script: + - service docker start - cd $GOPATH/src - mkdir -p gitlab.com/$CI_PROJECT_NAMESPACE - cd gitlab.com/$CI_PROJECT_NAMESPACE @@ -48,3 +52,16 @@ build-windows: paths: - dist +docker-build: + stage: build + script: + - make docker-build TAG=$CI_COMMIT_TAG + only: + - tags + +docker-push: + stage: publish + script: + - make docker-push TAG=$CI_COMMIT_TAG + only: + - tags \ No newline at end of file -- GitLab From 67a4bbb4236a9817158b8b6a1223108b83851702 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 21:52:05 +0200 Subject: [PATCH 08/35] Another try with the build --- build/ci/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index dcfa9e3..03b4f00 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -5,11 +5,7 @@ stages: - build - publish -services: - - docker:dind - before_script: - - service docker start - cd $GOPATH/src - mkdir -p gitlab.com/$CI_PROJECT_NAMESPACE - cd gitlab.com/$CI_PROJECT_NAMESPACE @@ -54,6 +50,8 @@ build-windows: docker-build: stage: build + services: + - docker:dind script: - make docker-build TAG=$CI_COMMIT_TAG only: @@ -61,6 +59,8 @@ docker-build: docker-push: stage: publish + services: + - docker:dind script: - make docker-push TAG=$CI_COMMIT_TAG only: -- GitLab From c3514d5bc324650dab1ce6238135c889c6a73136 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 22:20:19 +0200 Subject: [PATCH 09/35] Fix for Docker host --- build/ci/.gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index 03b4f00..4a8de9c 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -48,8 +48,10 @@ build-windows: paths: - dist -docker-build: +build-docker: stage: build + variables: + - DOCKER_HOST: docker services: - docker:dind script: @@ -57,8 +59,10 @@ docker-build: only: - tags -docker-push: +publish-docker: stage: publish + variables: + - DOCKER_HOST: docker services: - docker:dind script: -- GitLab From 6a7642c4ce71ec514fd4073874a7515a2c3f1d91 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 22:33:10 +0200 Subject: [PATCH 10/35] Still with docker build --- build/ci/.gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index 4a8de9c..6f8c2e4 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -66,6 +66,7 @@ publish-docker: services: - docker:dind script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - make docker-push TAG=$CI_COMMIT_TAG only: - tags \ No newline at end of file -- GitLab From 851fbe974ea4b76482eeec6f8b1e2365bb45aaf4 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 14 May 2018 22:38:26 +0200 Subject: [PATCH 11/35] Fix variables --- build/ci/.gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index 6f8c2e4..d8852c0 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -5,6 +5,9 @@ stages: - build - publish +variables: + DOCKER_HOST: docker + before_script: - cd $GOPATH/src - mkdir -p gitlab.com/$CI_PROJECT_NAMESPACE @@ -50,8 +53,6 @@ build-windows: build-docker: stage: build - variables: - - DOCKER_HOST: docker services: - docker:dind script: @@ -61,8 +62,6 @@ build-docker: publish-docker: stage: publish - variables: - - DOCKER_HOST: docker services: - docker:dind script: -- GitLab From e1fefef7f50e0f3143d8a407e4485b537405ea85 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 21:34:04 +0200 Subject: [PATCH 12/35] Use mutex for maintenance control --- internal/distill/stats.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/internal/distill/stats.go b/internal/distill/stats.go index 6a2bd89..95d1e44 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -216,15 +216,36 @@ func (s *Statistics) recGet() { } // runDbMaintenance -var maintenanceRunning = false +var ( + maintenanceMutex sync.Mutex + maintenanceRunning = false +) +// setRunMaintenance change maintenance status +func setRunMaintenance(val bool) { + maintenanceMutex.Lock() + defer maintenanceMutex.Unlock() + maintenanceRunning = val +} + +// isMaintenanceRunning check if there is already a routine doing maintenance +func isMaintenanceRunning() bool { + maintenanceMutex.Lock() + defer maintenanceMutex.Unlock() + return maintenanceRunning +} + +// runDbMaintenance runs the database maintenance +// TODO: add tests for this function func runDbMaintenance() { - if maintenanceRunning { + if isMaintenanceRunning() { return } - - maintenanceRunning = true + setRunMaintenance(true) + defer setRunMaintenance(false) wg.Add(1) + defer wg.Done() + // caluclate if gc is necessary deletes := globalStatistics.Deletes gcLimit := internal.Config.Tuning.DbGCDeletesCount @@ -248,14 +269,11 @@ func runDbMaintenance() { db.RunValueLogGC(internal.Config.Tuning.DbGCDiscardRation) mlog.Info("End maintenance n %d for deletes %d > %d", gcCount, deletes-latestGC, gcLimit) - // updaete the gcCount + // update the gcCount db.Update(func(txn *badger.Txn) (err error) { gcCount++ dbSetInt64(txn, sysKeyGCCount, gcCount) return }) - // unlock the maintenance - maintenanceRunning = false } - wg.Done() } -- GitLab From 9074d87bc060d5edf9b860325a34caefe95d6abd Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 21:37:02 +0200 Subject: [PATCH 13/35] Use gitlab registry instead of dockerhub --- build/ci/.gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/.gitlab-ci.yml b/build/ci/.gitlab-ci.yml index d8852c0..e1b6880 100644 --- a/build/ci/.gitlab-ci.yml +++ b/build/ci/.gitlab-ci.yml @@ -65,7 +65,7 @@ publish-docker: services: - docker:dind script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.gitlab.com - make docker-push TAG=$CI_COMMIT_TAG only: - tags \ No newline at end of file -- GitLab From b0fe140ae8b3ed02b9e758dbd24cf95b73217fe6 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 21:52:38 +0200 Subject: [PATCH 14/35] Move project to Welance OSS --- README.md | 4 ++-- cmd/backup.go | 2 +- cmd/import.go | 2 +- cmd/restore.go | 2 +- cmd/root.go | 2 +- cmd/start.go | 4 ++-- internal/distill/distill.go | 2 +- internal/distill/distill_test.go | 2 +- internal/distill/endpoints.go | 2 +- internal/distill/stats.go | 2 +- internal/distill/urlstore.go | 2 +- main.go | 2 +- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 05d6fcc..8f0d5be 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ Another url shortener -[![pipeline status](https://gitlab.com/welance/distill/badges/develop/pipeline.svg)](https://gitlab.com/welance/distill/commits/develop) +[![pipeline status](https://gitlab.com/welance/oss/distill/badges/develop/pipeline.svg)](https://gitlab.com/welance/oss/distill/commits/develop) -[![coverage report](https://gitlab.com/welance/distill/badges/develop/coverage.svg)](https://gitlab.com/welance/distill/commits/develop) +[![coverage report](https://gitlab.com/welance/oss/distill/badges/develop/coverage.svg)](https://gitlab.com/welance/oss/distill/commits/develop) ## Motivations diff --git a/cmd/backup.go b/cmd/backup.go index 4836f7a..dad3633 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -19,7 +19,7 @@ import ( "path/filepath" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/distill/internal/distill" + "gitlab.com/welance/oss/distill/internal/distill" "github.com/spf13/cobra" ) diff --git a/cmd/import.go b/cmd/import.go index 2186fb4..cfc264b 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -20,7 +20,7 @@ import ( "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/distill/internal/distill" + "gitlab.com/welance/oss/distill/internal/distill" ) // importCmd represents the import command diff --git a/cmd/restore.go b/cmd/restore.go index b43623b..be13f5e 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -20,7 +20,7 @@ import ( "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/distill/internal/distill" + "gitlab.com/welance/oss/distill/internal/distill" ) // restoreCmd represents the restore command diff --git a/cmd/root.go b/cmd/root.go index fc8a782..794e31a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,7 +22,7 @@ import ( "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" "github.com/spf13/viper" - "gitlab.com/welance/distill/internal" + "gitlab.com/welance/oss/distill/internal" ) var cfgFile, logFile, version string diff --git a/cmd/start.go b/cmd/start.go index 98b4da9..22f97e9 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -20,8 +20,8 @@ import ( "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/distill/internal" - "gitlab.com/welance/distill/internal/distill" + "gitlab.com/welance/oss/distill/internal/distill" + "gitlab.com/welance/oss/distill/internal" ) // startCmd represents the start command diff --git a/internal/distill/distill.go b/internal/distill/distill.go index 60af12d..1da0d11 100644 --- a/internal/distill/distill.go +++ b/internal/distill/distill.go @@ -12,7 +12,7 @@ import ( "github.com/jbrodriguez/mlog" gonanoid "github.com/matoous/go-nanoid" - "gitlab.com/welance/distill/internal" + "gitlab.com/welance/oss/distill/internal" ) // generateID generates a new id diff --git a/internal/distill/distill_test.go b/internal/distill/distill_test.go index 57204bb..c901575 100644 --- a/internal/distill/distill_test.go +++ b/internal/distill/distill_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/distill/internal" + "gitlab.com/welance/oss/distill/internal" ) func setupLog() { diff --git a/internal/distill/endpoints.go b/internal/distill/endpoints.go index 485d7b1..820f92a 100644 --- a/internal/distill/endpoints.go +++ b/internal/distill/endpoints.go @@ -7,7 +7,7 @@ import ( "time" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/distill/internal" + "gitlab.com/welance/oss/distill/internal" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" diff --git a/internal/distill/stats.go b/internal/distill/stats.go index 95d1e44..0219442 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -9,7 +9,7 @@ import ( "github.com/jbrodriguez/mlog" "github.com/dgraph-io/badger" - "gitlab.com/welance/distill/internal" + "gitlab.com/welance/oss/distill/internal" ) var ( diff --git a/internal/distill/urlstore.go b/internal/distill/urlstore.go index 3f76d94..1d400d5 100644 --- a/internal/distill/urlstore.go +++ b/internal/distill/urlstore.go @@ -10,7 +10,7 @@ import ( "github.com/bluele/gcache" "github.com/dgraph-io/badger" "github.com/jbrodriguez/mlog" - "gitlab.com/welance/distill/internal" + "gitlab.com/welance/oss/distill/internal" ) const ( diff --git a/main.go b/main.go index e119875..4266806 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ package main -import "gitlab.com/welance/distill/cmd" +import "gitlab.com/welance/oss/distill/cmd" // Version hold the version of the program var Version = "0.0.0" -- GitLab From 1baf1fa9748349401fa1e6ce3f2aec1d5ed9331a Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 21:57:30 +0200 Subject: [PATCH 15/35] Update LICENSE to MIT Add AUTHORS --- AUTHORS | 1 + LICENSE | 223 ++++++-------------------------------------------------- 2 files changed, 22 insertions(+), 202 deletions(-) create mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..701ce8d --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Andrea Giacobino \ No newline at end of file diff --git a/LICENSE b/LICENSE index d645695..03a148d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,202 +1,21 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +MIT License + +Copyright (c) 2018 Andrea Giacobino + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -- GitLab From a8cd5df04eec131cd18f99ac882400857a51ae3a Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 21:57:45 +0200 Subject: [PATCH 16/35] Fix Makefile --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index fe10c2b..7ea4644 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,11 @@ GOFILES = $(shell find . -name '*.go' -not -path './vendor/*') GOPACKAGES = $(shell go list ./... | grep -v /vendor/) -GIT_DESCR = $(shell git describe) +GIT_DESCR = $(shell git describe --always) # build output folder OUTPUTFOLDER = dist # docker image -DOCKERIMAGE = welance/distill -TAG = $(shell git describe) +DOCKER_IMAGE = welance/distill +DOCKER_TAG = $(shell git describe --always) # build paramters OS = linux ARCH = amd64 @@ -54,16 +54,17 @@ docker-build: build-dist @echo copy resources @cp configs/settings.docker.yaml $(OUTPUTFOLDER)/settings.yaml @echo build image - docker build -t $(DOCKERIMAGE):$(TAG) -f ./build/docker/Dockerfile . + docker build -t $(DOCKER_IMAGE):$(DOCKER_TAG) -f ./build/docker/Dockerfile . @echo done docker-push: docker-build @echo push image - docker push $(DOCKERIMAGE):$(TAG) + docker tag $(DOCKER_IMAGE):$(DOCKER_TAG) $(DOCKER_IMAGE):latest + docker push $(DOCKER_IMAGE) @echo done docker-run: - @docker run -p 1804:1804 $(DOCKERIMAGE) + @docker run -p 1804:1804 $(DOCKER_IMAGE) debug-start: @go run main.go -c configs/settings.sample.yaml --debug start -- GitLab From 6258d4619e8cebd7fa38c1f7427eeb0689abf8bb Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 22:34:31 +0200 Subject: [PATCH 17/35] Cosmetic changes for prime time --- cmd/root.go | 2 +- cmd/start.go | 3 ++- internal/distill/model.go | 12 ++++++------ internal/distill/stats.go | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 794e31a..1003d3e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -81,7 +81,7 @@ func initConfig() { // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { - mlog.Info("Using config file: %s", viper.ConfigFileUsed()) + mlog.Info("Using config file: %v", viper.ConfigFileUsed()) viper.Unmarshal(&internal.Config) internal.Config.Validate() } else { diff --git a/cmd/start.go b/cmd/start.go index 22f97e9..10d11cd 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -20,8 +20,8 @@ import ( "github.com/jbrodriguez/mlog" "github.com/spf13/cobra" - "gitlab.com/welance/oss/distill/internal/distill" "gitlab.com/welance/oss/distill/internal" + "gitlab.com/welance/oss/distill/internal/distill" ) // startCmd represents the start command @@ -62,6 +62,7 @@ func start(cmd *cobra.Command, args []string) { mlog.Info(" | (_| | \\__ \\ |_| | | |") mlog.Info(" \\__,_|_|___/\\__|_|_|_| v.%v", version) mlog.Info("") + mlog.Info("Listening to %v:%v", internal.Config.Server.Host, internal.Config.Server.Port) distill.NewSession() r := distill.RegisterEndpoints() diff --git a/internal/distill/model.go b/internal/distill/model.go index f0940e3..0dabaa2 100644 --- a/internal/distill/model.go +++ b/internal/distill/model.go @@ -61,15 +61,15 @@ type URLReq struct { // Statistics contains the global statistics type Statistics struct { - mutex sync.Mutex `json:"-"` - Urls int64 `json:"urls"` - Gets int64 `json:"gets"` - Upserts int64 `json:"upserts"` - Deletes int64 `json:"deletes"` + mutex sync.Mutex + Urls int64 `json:"urls"` + Gets int64 `json:"gets"` + Upserts int64 `json:"upserts"` + Deletes int64 `json:"deletes"` } func (s *Statistics) String() string { - return fmt.Sprintf("urls: %d, gets: %d, upserts:%d, deletes:%d", + return fmt.Sprintf("URLs: %d, GETs: %d, Inserts: %d, Deletes: %d", s.Urls, s.Gets, s.Upserts, diff --git a/internal/distill/stats.go b/internal/distill/stats.go index 0219442..c72cbcd 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -29,7 +29,7 @@ var ( // NewStatistics starts the statistics collector worker pool func NewStatistics() (err error) { - // initializae system key + // initialize system key sysKeyPurgeCount = keySys("ilij_sys_purge_count") sysKeyGCCount = keySys("ilij_sys_gc_count") // initialize stats keys @@ -81,7 +81,7 @@ func loadGlobalStatistics() (s *Statistics, err error) { return }) globalStatistics = s - mlog.Info("Statistics are %v", s) + mlog.Info("Status: %v", s) return } -- GitLab From 81b363ef09b2087a18c336dfeec5ee05b16c70bf Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sun, 10 Jun 2018 23:04:35 +0200 Subject: [PATCH 18/35] Disable database maintenance since is buggy --- internal/distill/stats.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/distill/stats.go b/internal/distill/stats.go index c72cbcd..f642247 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -187,7 +187,10 @@ func processEvents(workerID int) { } mlog.Trace("<<< Event pid: %v, opcode:%v %v", workerID, opcodeToString(uo.opcode), uo.ID) // run the maintenance - go runDbMaintenance() + // TODO: this has to be fixed since it can create issues with the database (hint: use a channel) + // it can happen that the database get closed while the maintenance is about to run + // creating a panic + // go runDbMaintenance() } // complete task mlog.Trace("Stop event processor id: %v", workerID) -- GitLab From 780cc98994406e6e3e94f58f4fbb20f2a25679a4 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 11 Jun 2018 00:14:51 +0200 Subject: [PATCH 19/35] Add defaults redirects in config files --- configs/settings.docker.yaml | 3 ++- configs/settings.sample.yaml | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/configs/settings.docker.yaml b/configs/settings.docker.yaml index 8fa47c9..a416f4f 100644 --- a/configs/settings.docker.yaml +++ b/configs/settings.docker.yaml @@ -3,7 +3,8 @@ server: host: "0.0.0.0" port: 1804 apiKey: 123123_changeme_changeme - rootRedirect: https://welance.com + rootRedirect: https://gitlab.com/welance/oss/distill/wikis/welcome + expiredRedirect: https://gitlab.com/welance/oss/distill/wikis/Expired-URL dbPath: /data/db enableStats: true diff --git a/configs/settings.sample.yaml b/configs/settings.sample.yaml index 7963144..197bbbc 100644 --- a/configs/settings.sample.yaml +++ b/configs/settings.sample.yaml @@ -2,8 +2,9 @@ server: host: "0.0.0.0" port: 1804 - apiKey: naiuraoijodmaojorierjaomdaodorpaokda - rootRedirect: https://mywebsite.com + apiKey: 123123_changeme_changeme + rootRedirect: https://gitlab.com/welance/oss/distill/wikis/welcome + expiredRedirect: https://gitlab.com/welance/oss/distill/wikis/Expired-URL dbPath: ./distill.db enableStats: true -- GitLab From f02977ff72894695557ab985d0ab78fe0e319932 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 11 Jun 2018 00:15:13 +0200 Subject: [PATCH 20/35] Add defaults for all values of configuration --- internal/config.go | 50 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/internal/config.go b/internal/config.go index e912f2a..279c712 100644 --- a/internal/config.go +++ b/internal/config.go @@ -45,17 +45,38 @@ type ConfigSchema struct { Tuning TuningConfig `yaml:"tuning"` } -//Validate configuration -func (c *ConfigSchema) Validate() { +func empty(s string) bool { + return len(strings.TrimSpace(s)) == 0 +} - if c.ShortID.Length < 3 { - panic("short_id.length must be at least 3") +//Defaults generate configuration defaults +func (c *ConfigSchema) Defaults() { + // for server + if empty(c.Server.Host) { + c.Server.Host = "0.0.0.0" + } + if c.Server.Port == 0 { + c.Server.Port = 1804 + } + if empty(c.Server.DbPath) { + c.Server.DbPath = "distill.db" + } + if empty(c.Server.RootRedirect) { + c.Server.RootRedirect = "https://gitlab.com/welance/oss/distill/wikis/welcome" + } + if empty(c.Server.ExpiredRedirect) { + c.Server.ExpiredRedirect = "https://gitlab.com/welance/oss/distill/wikis/Expired-URL" } - if len(c.ShortID.Alphabet) < c.ShortID.Length { - panic(fmt.Sprint("short_id.alphabet must be at least ", c.ShortID.Length, " characters long")) + // for short id + if empty(c.ShortID.Alphabet) { + c.ShortID.Alphabet = "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" + } + if c.ShortID.Length == 0 { + c.ShortID.Length = 6 } + // For tuning if c.Tuning.StatsEventsQueueSize <= 0 { c.Tuning.StatsEventsQueueSize = 2048 } @@ -88,10 +109,25 @@ func (c *ConfigSchema) Validate() { c.Tuning.BckCSVIterPrefetchSize = 2048 } - if len(strings.TrimSpace(c.Tuning.APIKeyHeaderName)) == 0 { + if empty(c.Tuning.APIKeyHeaderName) { c.Tuning.APIKeyHeaderName = "X-API-KEY" } +} +//Validate configuration +func (c *ConfigSchema) Validate() { + + if empty(c.Server.APIKey) { + panic("server.apy_key cannot be empty") + } + + if c.ShortID.Length < 3 { + panic("short_id.length must be at least 3") + } + + if len(c.ShortID.Alphabet) < c.ShortID.Length { + panic(fmt.Sprint("short_id.alphabet must be at least ", c.ShortID.Length, " characters long")) + } } // Config sytem configuration -- GitLab From b1dc3d14ed1499c35591ce70cd1070606ca9e911 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 11 Jun 2018 22:48:15 +0200 Subject: [PATCH 21/35] Add helpers functions and tests --- internal/config.go | 10 +-- internal/distill/distill.go | 4 + pkg/common/helpers.go | 48 ++++++++++++ pkg/common/helpers_test.go | 149 ++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 6 deletions(-) create mode 100644 pkg/common/helpers.go create mode 100644 pkg/common/helpers_test.go diff --git a/internal/config.go b/internal/config.go index 279c712..9f905ab 100644 --- a/internal/config.go +++ b/internal/config.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" "time" + + "gitlab.com/welance/oss/distill/pkg/common" ) // ServerConfig configuration for the server @@ -52,15 +54,11 @@ func empty(s string) bool { //Defaults generate configuration defaults func (c *ConfigSchema) Defaults() { // for server - if empty(c.Server.Host) { - c.Server.Host = "0.0.0.0" - } + common.DefaultIfEmpty(&c.Server.Host, "0.0.0.0") if c.Server.Port == 0 { c.Server.Port = 1804 } - if empty(c.Server.DbPath) { - c.Server.DbPath = "distill.db" - } + common.DefaultIfEmpty(&c.Server.DbPath, "distill.db") if empty(c.Server.RootRedirect) { c.Server.RootRedirect = "https://gitlab.com/welance/oss/distill/wikis/welcome" } diff --git a/internal/distill/distill.go b/internal/distill/distill.go index 1da0d11..5923f68 100644 --- a/internal/distill/distill.go +++ b/internal/distill/distill.go @@ -15,6 +15,10 @@ import ( "gitlab.com/welance/oss/distill/internal" ) +func randomString(alphabet string, length int) (string, error) { + return gonanoid.Generate(alphabet, length) +} + // generateID generates a new id func generateID() (shortID string) { a := internal.Config.ShortID.Alphabet diff --git a/pkg/common/helpers.go b/pkg/common/helpers.go new file mode 100644 index 0000000..3cea2f9 --- /dev/null +++ b/pkg/common/helpers.go @@ -0,0 +1,48 @@ +package common + +import ( + "fmt" + "strings" + + gonanoid "github.com/matoous/go-nanoid" +) + +// IsEmptyStr tells if a string is empty or not +func IsEmptyStr(s string) bool { + return len(strings.TrimSpace(s)) == 0 +} + +// DefaultIfEmptyStr set a default for a string if it is nulled +func DefaultIfEmptyStr(s *string, defaultS string) { + if IsEmptyStr(*s) { + *s = defaultS + } +} + +// DefaultIfEmptyInt set the value of an int to a default if it is nulled (0) +func DefaultIfEmptyInt(v *int, defaultV int) { + if *v <= 0 { + *v = defaultV + } +} + +// RandomString generate a random string of required lenght an with requested alphabet +func RandomString(alphabet string, length int) (s string, err error) { + if IsEmptyStr(alphabet) { + err = fmt.Errorf("alphabet must not be empty") + return + } + if length <= 0 { + err = fmt.Errorf("string length must be longer than 0") + return + } + return gonanoid.Generate(alphabet, length) +} + +// GenerateSecret generate a string that can be used as secrete api key +func GenerateSecret() string { + a := "asdfghjklqwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM1234567890#!-" + l := 50 + secret, _ := RandomString(a, l) + return secret +} diff --git a/pkg/common/helpers_test.go b/pkg/common/helpers_test.go new file mode 100644 index 0000000..97f1604 --- /dev/null +++ b/pkg/common/helpers_test.go @@ -0,0 +1,149 @@ +package common + +import "testing" + +func TestIsEmptyStr(t *testing.T) { + type args struct { + } + tests := []struct { + name string + s string + want bool + }{ + {"ok", "ciao", false}, + {"space", " ", true}, + {"spaces", " ", true}, + {"one char", " 1 ", false}, + {"one char space right", "1 ", false}, + {"one char space right", " 1", false}, + {"with tabs", "\t\t ", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsEmptyStr(tt.s); got != tt.want { + t.Errorf("IsEmptyStr() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestDefaultIfEmptyStr(t *testing.T) { + type args struct { + s string + defaultS string + } + tests := []struct { + name string + args args + wantDefault bool + }{ + {"no default", args{"foo", "bar"}, false}, + {"yes default", args{"", "bar"}, true}, + {"yes default", args{"\t\t", "bar"}, true}, + {"yes default", args{"\ta\t", "bar"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + DefaultIfEmptyStr(&tt.args.s, tt.args.defaultS) + if tt.wantDefault && tt.args.s != tt.args.defaultS { + t.Errorf("DefaultIfEmptyStr expected %v got %v", tt.args.defaultS, tt.args.s) + } + if !tt.wantDefault && tt.args.s == tt.args.defaultS { + t.Errorf("DefaultIfEmptyStr expected %v got %v", tt.args.s, tt.args.defaultS) + } + }) + } +} + +func TestDefaultIfEmptyInt(t *testing.T) { + type args struct { + v int + defaultV int + } + tests := []struct { + name string + args args + wantDefault bool + }{ + {"no default", args{10, 1000}, false}, + {"no default", args{50000, 1011}, false}, + {"yes default", args{0, 1002}, true}, + {"yes default", args{-100, 1010}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + DefaultIfEmptyInt(&tt.args.v, tt.args.defaultV) + if tt.wantDefault && tt.args.v != tt.args.defaultV { + t.Errorf("DefaultIfEmptyStr expected %v got %v", tt.args.defaultV, tt.args.v) + } + if !tt.wantDefault && tt.args.v == tt.args.defaultV { + t.Errorf("DefaultIfEmptyStr expected %v got %v", tt.args.v, tt.args.defaultV) + } + }) + } +} + +func TestRandomString(t *testing.T) { + type args struct { + alphabet string + length int + } + tests := []struct { + name string + args args + wantLen int + wantErr bool + }{ + {"ok 1", args{"abcdefg123", 10}, 10, false}, + {"ok 2", args{"abcdefg123", 100}, 100, false}, + {"ok 3", args{"abcdefg123", 13}, 13, false}, + {"not ok", args{"", 13}, 13, true}, + {"not ok", args{" ", 13}, 13, true}, + {"not ok", args{"\t\t", 13}, 13, true}, + {"not ok", args{"\t\t", -13}, 13, true}, + {"not ok", args{"asdfghjk", -1}, 13, true}, + {"not ok", args{"asdfghjk", 0}, 131, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := RandomString(tt.args.alphabet, tt.args.length) + t.Log(err) + if (err != nil) != tt.wantErr { + t.Errorf("RandomString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && len(got) != tt.wantLen { + t.Errorf("RandomString() = %v, want %v", len(got), tt.wantLen) + } + }) + } +} + +func TestGenerateSecret(t *testing.T) { + tests := []struct { + name string + wantLen int + }{ + {"test", 50}, + {"test", 50}, + {"test", 50}, + {"test", 50}, + {"test", 50}, + {"test", 50}, + {"test", 50}, + {"test", 50}, + } + for _, tt := range tests { + oldOne := "" + t.Run(tt.name, func(t *testing.T) { + got := GenerateSecret() + if len(got) != tt.wantLen { + t.Errorf("GenerateSecret() = %v, want %v", len(got), tt.wantLen) + } + if got == oldOne { + t.Errorf("GenerateSecret() should be fairly unique") + } + oldOne = got + }) + } +} -- GitLab From 4ee7e66122f6b462f84605fcd5c81d83447ee693 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 11 Jun 2018 22:48:49 +0200 Subject: [PATCH 22/35] Fix config and make it more concise --- internal/config.go | 69 ++++++++++++---------------------------------- 1 file changed, 18 insertions(+), 51 deletions(-) diff --git a/internal/config.go b/internal/config.go index 9f905ab..af0a89c 100644 --- a/internal/config.go +++ b/internal/config.go @@ -32,8 +32,8 @@ type TuningConfig struct { StatsEventsWorkerNum int `yaml:"statsEventsWorkerNum"` StatsEventsQueueSize int `yaml:"statsEventsQueueSize"` StatsCaheSize int `yaml:"statsCacheSize"` - DbPurgeWritesCount int64 `yaml:"dbPurgeWritesCount"` - DbGCDeletesCount int64 `yaml:"dbGCDeletesCount"` + DbPurgeWritesCount int `yaml:"dbPurgeWritesCount"` + DbGCDeletesCount int `yaml:"dbGCDeletesCount"` DbGCDiscardRation float64 `yaml:"dbGCDiscardRation"` URLCaheSize int `yaml:"URLCaheSize"` BckCSVIterPrefetchSize int `yaml:"exportIteratorPrefetchSize"` @@ -54,68 +54,35 @@ func empty(s string) bool { //Defaults generate configuration defaults func (c *ConfigSchema) Defaults() { // for server - common.DefaultIfEmpty(&c.Server.Host, "0.0.0.0") - if c.Server.Port == 0 { - c.Server.Port = 1804 - } - common.DefaultIfEmpty(&c.Server.DbPath, "distill.db") - if empty(c.Server.RootRedirect) { - c.Server.RootRedirect = "https://gitlab.com/welance/oss/distill/wikis/welcome" - } - if empty(c.Server.ExpiredRedirect) { - c.Server.ExpiredRedirect = "https://gitlab.com/welance/oss/distill/wikis/Expired-URL" - } + common.DefaultIfEmptyStr(&c.Server.Host, "0.0.0.0") + common.DefaultIfEmptyInt(&c.Server.Port, 1804) + common.DefaultIfEmptyStr(&c.Server.DbPath, "distill.db") + common.DefaultIfEmptyStr(&c.Server.RootRedirect, "https://gitlab.com/welance/oss/distill/wikis/welcome") + common.DefaultIfEmptyStr(&c.Server.ExpiredRedirect, "https://gitlab.com/welance/oss/distill/wikis/Expired-URL") // for short id - if empty(c.ShortID.Alphabet) { - c.ShortID.Alphabet = "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" - } - if c.ShortID.Length == 0 { - c.ShortID.Length = 6 - } + common.DefaultIfEmptyStr(&c.ShortID.Alphabet, "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789") + common.DefaultIfEmptyInt(&c.ShortID.Length, 6) // For tuning - if c.Tuning.StatsEventsQueueSize <= 0 { - c.Tuning.StatsEventsQueueSize = 2048 - } - - if c.Tuning.StatsEventsWorkerNum <= 0 { - c.Tuning.StatsEventsWorkerNum = 1 - } - - if c.Tuning.StatsCaheSize <= 0 { - c.Tuning.StatsCaheSize = 1024 - } - - if c.Tuning.DbPurgeWritesCount <= 0 { - c.Tuning.DbPurgeWritesCount = 2000 - } - - if c.Tuning.DbGCDeletesCount <= 0 { - c.Tuning.DbGCDeletesCount = 500 - } - + common.DefaultIfEmptyInt(&c.Tuning.StatsEventsQueueSize, 2048) + common.DefaultIfEmptyInt(&c.Tuning.StatsEventsWorkerNum, 1) + common.DefaultIfEmptyInt(&c.Tuning.StatsCaheSize, 1024) + common.DefaultIfEmptyInt(&c.Tuning.DbPurgeWritesCount, 2000) + common.DefaultIfEmptyInt(&c.Tuning.DbGCDeletesCount, 500) if c.Tuning.DbGCDiscardRation <= 0 || c.Tuning.DbGCDiscardRation > 1 { c.Tuning.DbGCDiscardRation = 0.5 } + common.DefaultIfEmptyInt(&c.Tuning.URLCaheSize, 2048) + common.DefaultIfEmptyInt(&c.Tuning.BckCSVIterPrefetchSize, 2048) + common.DefaultIfEmptyStr(&c.Tuning.APIKeyHeaderName, "X-API-KEY") - if c.Tuning.URLCaheSize <= 0 || c.Tuning.URLCaheSize > 1 { - c.Tuning.URLCaheSize = 2048 - } - - if c.Tuning.BckCSVIterPrefetchSize <= 0 || c.Tuning.BckCSVIterPrefetchSize > 1 { - c.Tuning.BckCSVIterPrefetchSize = 2048 - } - - if empty(c.Tuning.APIKeyHeaderName) { - c.Tuning.APIKeyHeaderName = "X-API-KEY" - } } //Validate configuration func (c *ConfigSchema) Validate() { - if empty(c.Server.APIKey) { + if common.IsEmptyStr(c.Server.APIKey) { panic("server.apy_key cannot be empty") } -- GitLab From 180454fd6065d61680f4cadaf72928fd875996a3 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 11 Jun 2018 22:49:21 +0200 Subject: [PATCH 23/35] Use helpers functions --- internal/distill/distill.go | 8 ++------ internal/distill/distill_test.go | 10 ++++++++++ internal/distill/stats.go | 14 +++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/internal/distill/distill.go b/internal/distill/distill.go index 5923f68..5e6a304 100644 --- a/internal/distill/distill.go +++ b/internal/distill/distill.go @@ -11,20 +11,16 @@ import ( "time" "github.com/jbrodriguez/mlog" - gonanoid "github.com/matoous/go-nanoid" "gitlab.com/welance/oss/distill/internal" + "gitlab.com/welance/oss/distill/pkg/common" ) -func randomString(alphabet string, length int) (string, error) { - return gonanoid.Generate(alphabet, length) -} - // generateID generates a new id func generateID() (shortID string) { a := internal.Config.ShortID.Alphabet l := internal.Config.ShortID.Length // a and l are validated before - shortID, _ = gonanoid.Generate(a, l) + shortID, _ = common.RandomString(a, l) return } diff --git a/internal/distill/distill_test.go b/internal/distill/distill_test.go index c901575..0441ba0 100644 --- a/internal/distill/distill_test.go +++ b/internal/distill/distill_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "gitlab.com/welance/oss/distill/pkg/common" + "github.com/jbrodriguez/mlog" "gitlab.com/welance/oss/distill/internal" ) @@ -28,12 +30,14 @@ func buildConifgTest() { internal.Config = internal.ConfigSchema{ Server: internal.ServerConfig{ DbPath: path, + APIKey: common.GenerateSecret(), }, ShortID: internal.ShortIDConfig{ Alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", Length: 6, }, } + internal.Config.Defaults() internal.Config.Validate() } @@ -44,12 +48,14 @@ func buildConifgPanicTest() { internal.Config = internal.ConfigSchema{ Server: internal.ServerConfig{ DbPath: path, + APIKey: common.GenerateSecret(), }, ShortID: internal.ShortIDConfig{ Alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", Length: 6, }, } + internal.Config.Defaults() internal.Config.Validate() } @@ -60,6 +66,7 @@ func buildConifgTestShortIDParams(alphabet string, length int) { internal.Config = internal.ConfigSchema{ Server: internal.ServerConfig{ DbPath: path, + APIKey: common.GenerateSecret(), }, ShortID: internal.ShortIDConfig{ Alphabet: alphabet, @@ -71,6 +78,7 @@ func buildConifgTestShortIDParams(alphabet string, length int) { StatsEventsWorkerNum: 2, }, } + internal.Config.Defaults() internal.Config.Validate() } @@ -81,6 +89,7 @@ func buildConifgTestExpireParams(ttl, maxr int64, expire time.Time) { internal.Config = internal.ConfigSchema{ Server: internal.ServerConfig{ DbPath: path, + APIKey: common.GenerateSecret(), }, ShortID: internal.ShortIDConfig{ Alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789", @@ -93,6 +102,7 @@ func buildConifgTestExpireParams(ttl, maxr int64, expire time.Time) { StatsEventsWorkerNum: 20, }, } + internal.Config.Defaults() internal.Config.Validate() } diff --git a/internal/distill/stats.go b/internal/distill/stats.go index f642247..45ea832 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -252,30 +252,30 @@ func runDbMaintenance() { // caluclate if gc is necessary deletes := globalStatistics.Deletes gcLimit := internal.Config.Tuning.DbGCDeletesCount - gcCount := int64(0) + gcCount := 0 // retrieve the gcCount from the db db.View(func(txn *badger.Txn) (err error) { - gcCount = dbGetInt64(txn, sysKeyGCCount) + gcCount = int(dbGetInt64(txn, sysKeyGCCount)) return }) latestGC := gcCount * gcLimit - if latestGC > deletes { + if latestGC > int(deletes) { // there was a reset should reset in the stats gcCount, latestGC = 0, 0 } - if deletes-latestGC > gcLimit { - mlog.Info("Start maintenance n %d for deletes %d > %d", gcCount, deletes-latestGC, gcLimit) + if int(deletes)-latestGC > gcLimit { + mlog.Info("Start maintenance n %d for deletes %d > %d", gcCount, int(deletes)-latestGC, gcLimit) mlog.Info("") db.RunValueLogGC(internal.Config.Tuning.DbGCDiscardRation) - mlog.Info("End maintenance n %d for deletes %d > %d", gcCount, deletes-latestGC, gcLimit) + mlog.Info("End maintenance n %d for deletes %d > %d", gcCount, int(deletes)-latestGC, gcLimit) // update the gcCount db.Update(func(txn *badger.Txn) (err error) { gcCount++ - dbSetInt64(txn, sysKeyGCCount, gcCount) + dbSetInt64(txn, sysKeyGCCount, int64(gcCount)) return }) } -- GitLab From d67560af887853a30dd526917d311bb60bceb319 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 11 Jun 2018 22:50:23 +0200 Subject: [PATCH 24/35] Update to badger 1.5.1 --- Gopkg.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 23f81ce..eb7e446 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -16,8 +16,8 @@ [[projects]] name = "github.com/dgraph-io/badger" packages = [".","options","protos","skl","table","y"] - revision = "3ff1bfbb11c02c78c11269096669396c2d2cbbd3" - version = "v1.5.0" + revision = "690400e629977977ecddfa8ab3a7f9285e1041a8" + version = "v1.5.1" [[projects]] branch = "master" @@ -160,6 +160,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b4e4735f721cf82b5902fa88cd11db3500366ebd67c3343b58a1555b3bdd45ba" + inputs-digest = "dc7958cef7ce29cd96456bdb0043a62dd32fe48ae9443db2d9a45cec739f0a6e" solver-name = "gps-cdcl" solver-version = 1 -- GitLab From 5993615af30cfd91aa3fc1c7983c3f921060bcae Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 12 Jun 2018 00:09:24 +0200 Subject: [PATCH 25/35] Ask to generate a configuration if one is not found --- cmd/root.go | 23 +++++++++++++++++++++-- cmd/start.go | 13 +++---------- internal/config.go | 20 ++++++++++++++++++++ pkg/common/helpers.go | 24 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 1003d3e..fc1f9f7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -23,10 +23,11 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "gitlab.com/welance/oss/distill/internal" + "gitlab.com/welance/oss/distill/pkg/common" ) var cfgFile, logFile, version string -var profile, debug bool +var profile, debug, generateConfigOnly bool // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ @@ -56,6 +57,8 @@ func init() { RootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is /etc/distill/settings.yaml)") RootCmd.PersistentFlags().StringVar(&logFile, "log", "", "set a logging file, default stdout") RootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debug") + RootCmd.PersistentFlags().BoolVar(&generateConfigOnly, "generate-config", false, "print the default configuration file and exit") + } // initConfig reads in config file and ENV variables if set. @@ -69,10 +72,16 @@ func initConfig() { mlog.Start(loglevel, logFile) mlog.DefaultFlags = log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile + // if only generate config + if generateConfigOnly { + + } + // set configuration paramteres viper.SetConfigName("settings") // name of config file (without extension) viper.AddConfigPath("/etc/distill") // adding home directory as first search path viper.AddConfigPath("./configs") + viper.AddConfigPath(".") viper.AutomaticEnv() // read in environment variables that match // if there is the config file read it if len(cfgFile) > 0 { // enable ability to specify config file via flag @@ -83,8 +92,18 @@ func initConfig() { if err := viper.ReadInConfig(); err == nil { mlog.Info("Using config file: %v", viper.ConfigFileUsed()) viper.Unmarshal(&internal.Config) + internal.Config.Defaults() internal.Config.Validate() } else { - mlog.Fatalf("Configuration file not found!! %v", cfgFile) + switch err.(type) { + case viper.ConfigFileNotFoundError: + if do := common.AskYes("A configuration file was not found, would you like to generate one?", true); do { + internal.GenerateDefaultConfig("settings.yaml", version) + fmt.Println("Configuration settings.yaml created") + return + } + } + fmt.Println("Configuration file not found!!") + os.Exit(1) } } diff --git a/cmd/start.go b/cmd/start.go index 10d11cd..1867055 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -27,14 +27,9 @@ import ( // startCmd represents the start command var startCmd = &cobra.Command{ Use: "start", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: start, + Short: "Start distill", + Long: ``, + Run: start, } func init() { @@ -53,8 +48,6 @@ func init() { } func start(cmd *cobra.Command, args []string) { - // TODO: Work your own magic here - mlog.Info(" _ _ _ _ _ _ ") mlog.Info(" | (_) | | (_) | |") mlog.Info(" __| |_ ___| |_ _| | |") diff --git a/internal/config.go b/internal/config.go index af0a89c..f60b7af 100644 --- a/internal/config.go +++ b/internal/config.go @@ -2,10 +2,12 @@ package internal import ( "fmt" + "io/ioutil" "strings" "time" "gitlab.com/welance/oss/distill/pkg/common" + yaml "gopkg.in/yaml.v2" ) // ServerConfig configuration for the server @@ -97,3 +99,21 @@ func (c *ConfigSchema) Validate() { // Config sytem configuration var Config ConfigSchema + +// GenerateDefaultConfig generate a default configuration file an writes it in the outFile +func GenerateDefaultConfig(outFile, version string) { + Config.Defaults() + Config.Server.APIKey = common.GenerateSecret() + b, _ := yaml.Marshal(Config) + data := strings.Join([]string{ + "#", + fmt.Sprintf("# Default configuration for Distill v%s", version), + "# http://gitlab.com/welance/oss/distill", + "#\n", + fmt.Sprintf("%s", b), + "#", + "# Config end", + "#", + }, "\n") + ioutil.WriteFile(outFile, []byte(data), 0600) +} diff --git a/pkg/common/helpers.go b/pkg/common/helpers.go index 3cea2f9..771135d 100644 --- a/pkg/common/helpers.go +++ b/pkg/common/helpers.go @@ -1,12 +1,19 @@ package common import ( + "bufio" "fmt" + "os" "strings" gonanoid "github.com/matoous/go-nanoid" ) +// IsEqStr tells if two strings a and b are equals after trimming spaces +func IsEqStr(a, b string) bool { + return strings.TrimSpace(a) == strings.TrimSpace(b) +} + // IsEmptyStr tells if a string is empty or not func IsEmptyStr(s string) bool { return len(strings.TrimSpace(s)) == 0 @@ -46,3 +53,20 @@ func GenerateSecret() string { secret, _ := RandomString(a, l) return secret } + +// AskYes prompt a yes/no question to the prompt +func AskYes(question string, defaultYes bool) (isYes bool) { + fmt.Print(question) + if defaultYes { + fmt.Print(" [yes]: ") + } else { + fmt.Print(" [no]: ") + } + reader := bufio.NewReader(os.Stdin) + reply, _ := reader.ReadString('\n') + DefaultIfEmptyStr(&reply, "yes") + if IsEqStr(reply, "yes") { + return true + } + return +} -- GitLab From df65dd833bf46fcc4c3a91870bcd859baf1ad0e6 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 12 Jun 2018 00:32:07 +0200 Subject: [PATCH 26/35] Add examples for settings and docker-compose.yml Remove unused sample settings --- configs/settings.sample.yaml | 18 ------------------ examples/docker-compose.yml | 13 +++++++++++++ examples/settings.yaml | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 18 deletions(-) delete mode 100644 configs/settings.sample.yaml create mode 100644 examples/docker-compose.yml create mode 100644 examples/settings.yaml diff --git a/configs/settings.sample.yaml b/configs/settings.sample.yaml deleted file mode 100644 index 197bbbc..0000000 --- a/configs/settings.sample.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# server configuration -server: - host: "0.0.0.0" - port: 1804 - apiKey: 123123_changeme_changeme - rootRedirect: https://gitlab.com/welance/oss/distill/wikis/welcome - expiredRedirect: https://gitlab.com/welance/oss/distill/wikis/Expired-URL - dbPath: ./distill.db - enableStats: true - -# short id configuration -shortId: - alphabet: "abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789" - length: 6 - ttl: 0 # 0 means no expiration - maxRequests: 10 # 0 means no limits - - diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml new file mode 100644 index 0000000..d106ab9 --- /dev/null +++ b/examples/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.1' +services: + clerk: + image: registry.gitlab.com/welance/oss/distill:latest + container_name: distill + restart: unless-stopped + networks: + - local + ports: + - 1804:1804 + volumes: + - ./distil.db:/data/distill.db + - ./settings.yml:/data/configs/settings.docker.yaml diff --git a/examples/settings.yaml b/examples/settings.yaml new file mode 100644 index 0000000..1962ede --- /dev/null +++ b/examples/settings.yaml @@ -0,0 +1,32 @@ +# +# Default configuration for Distill v0.0.3-ci-14-gd67560a +# http://gitlab.com/welance/oss/distill +# + +server: + apiKey: sSftPgEULMt4UWMh8idQgaTiJh9dKlXfdeiB5qcoCxc!ZehVik + host: 0.0.0.0 + port: 1804 + dbPath: distill.db + rootRedirect: https://gitlab.com/welance/oss/distill/wikis/welcome + expiredRedirect: https://gitlab.com/welance/oss/distill/wikis/Expired-URL +shortId: + alphabet: abcdefghkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789 + length: 6 + maxRequests: 0 + ttl: 0 + expireOn: 0001-01-01T00:00:00Z +tuning: + statsEventsWorkerNum: 1 + statsEventsQueueSize: 2048 + statsCacheSize: 1024 + dbPurgeWritesCount: 2000 + dbGCDeletesCount: 500 + dbGCDiscardRation: 0.5 + URLCaheSize: 2048 + exportIteratorPrefetchSize: 2048 + apiKeyHeaderName: X-API-KEY + +# +# Config end +# \ No newline at end of file -- GitLab From 146ed2e4571dcf43650c3408d177b0ed6a12950b Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 12 Jun 2018 01:03:53 +0200 Subject: [PATCH 27/35] common method for string comparison also does lowercasing --- pkg/common/helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/helpers.go b/pkg/common/helpers.go index 771135d..8150912 100644 --- a/pkg/common/helpers.go +++ b/pkg/common/helpers.go @@ -9,9 +9,9 @@ import ( gonanoid "github.com/matoous/go-nanoid" ) -// IsEqStr tells if two strings a and b are equals after trimming spaces +// IsEqStr tells if two strings a and b are equals after trimming spaces and lowercasing func IsEqStr(a, b string) bool { - return strings.TrimSpace(a) == strings.TrimSpace(b) + return strings.ToLower(strings.TrimSpace(a)) == strings.ToLower(strings.TrimSpace(b)) } // IsEmptyStr tells if a string is empty or not -- GitLab From 2d86c07dd1db0ee31cba0bb563cd70cde4e9e819 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 12 Jun 2018 01:04:23 +0200 Subject: [PATCH 28/35] Use common pkg for string comparison --- internal/distill/distill.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/distill/distill.go b/internal/distill/distill.go index 5e6a304..bb6b7ce 100644 --- a/internal/distill/distill.go +++ b/internal/distill/distill.go @@ -175,7 +175,7 @@ func ImportCSV(inFile string) (rows int, err error) { mlog.Error(err) break } - if rows == 0 && record[0] == "url" { + if rows == 0 && common.IsEqStr(record[0], "url") { // header, skip continue } -- GitLab From a210fc209373ed16a437f4b0838baac5ee384607 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 12 Jun 2018 01:04:50 +0200 Subject: [PATCH 29/35] Fix Makefile docker image name --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7ea4644..704e4a0 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GIT_DESCR = $(shell git describe --always) # build output folder OUTPUTFOLDER = dist # docker image -DOCKER_IMAGE = welance/distill +DOCKER_IMAGE = registry.gitlab.com/welance/oss/distill DOCKER_TAG = $(shell git describe --always) # build paramters OS = linux -- GitLab From 17574880bb43e94fa95b19d6c431e381a7526b90 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 12 Jun 2018 01:05:19 +0200 Subject: [PATCH 30/35] Add docker-compose.yml example and csv for import/export examples --- examples/csv/import.urls.csv | 502 +++++++++++++ examples/csv/import.urls.with.ids.csv | 1002 +++++++++++++++++++++++++ examples/docker-compose.yml | 15 +- 3 files changed, 1514 insertions(+), 5 deletions(-) create mode 100644 examples/csv/import.urls.csv create mode 100644 examples/csv/import.urls.with.ids.csv diff --git a/examples/csv/import.urls.csv b/examples/csv/import.urls.csv new file mode 100644 index 0000000..d718a12 --- /dev/null +++ b/examples/csv/import.urls.csv @@ -0,0 +1,502 @@ +URL +https://facebook.com +https://twitter.com +https://google.com +https://youtube.com +https://instagram.com +https://linkedin.com +https://wordpress.org +https://pinterest.com +https://wikipedia.org +https://wordpress.com +https://blogspot.com +https://apple.com +https://adobe.com +https://tumblr.com +https://youtu.be +https://amazon.com +https://goo.gl +https://vimeo.com +https://flickr.com +https://microsoft.com +https://yahoo.com +https://godaddy.com +https://qq.com +https://bit.ly +https://vk.com +https://reddit.com +https://w3.org +https://baidu.com +https://nytimes.com +https://t.co +https://europa.eu +https://buydomains.com +https://wp.com +https://statcounter.com +https://miitbeian.gov.cn +https://jimdo.com +https://blogger.com +https://github.com +https://weebly.com +https://soundcloud.com +https://mozilla.org +https://bbc.co.uk +https://yandex.ru +https://myspace.com +https://google.de +https://addthis.com +https://nih.gov +https://theguardian.com +https://google.co.jp +https://cnn.com +https://stumbleupon.com +https://gravatar.com +https://digg.com +https://addtoany.com +https://creativecommons.org +https://paypal.com +https://yelp.com +https://imdb.com +https://huffingtonpost.com +https://feedburner.com +https://issuu.com +https://wixsite.com +https://wix.com +https://dropbox.com +https://forbes.com +https://miibeian.gov.cn +https://amazonaws.com +https://google.co.uk +https://washingtonpost.com +https://bluehost.com +https://etsy.com +https://go.com +https://msn.com +https://wsj.com +https://ameblo.jp +https://archive.org +https://slideshare.net +https://e-recht24.de +https://weibo.com +https://fc2.com +https://eventbrite.com +https://parallels.com +https://doubleclick.net +https://mail.ru +https://sourceforge.net +https://amazon.co.uk +https://telegraph.co.uk +https://ebay.com +https://amzn.to +https://livejournal.com +https://51.la +https://free.fr +https://yahoo.co.jp +https://dailymail.co.uk +https://reuters.com +https://taobao.com +https://wikimedia.org +https://amazon.de +https://typepad.com +https://hatena.ne.jp +https://bloomberg.com +https://elegantthemes.com +https://eepurl.com +https://usatoday.com +https://about.com +https://medium.com +https://macromedia.com +https://xing.com +https://bing.com +https://time.com +https://www.gov.uk +https://google.it +https://cdc.gov +https://tripadvisor.com +https://cpanel.net +https://amazon.co.jp +https://npr.org +https://harvard.edu +https://bbb.org +https://aol.com +https://constantcontact.com +https://latimes.com +https://icio.us +https://list-manage.com +https://webs.com +https://opera.com +https://beian.gov.cn +https://vkontakte.ru +https://blogspot.co.uk +https://live.com +https://bandcamp.com +https://apache.org +https://bbc.com +https://businessinsider.com +https://dailymotion.com +https://cpanel.com +https://disqus.com +https://behance.net +https://mit.edu +https://rambler.ru +https://gnu.org +https://sina.com.cn +https://spotify.com +https://joomla.org +https://google.es +https://line.me +https://wired.com +https://github.io +https://stanford.edu +https://googleusercontent.com +https://kickstarter.com +https://guardian.co.uk +https://imgur.com +https://google.fr +https://goodreads.com +https://nasa.gov +https://rakuten.co.jp +https://surveymonkey.com +https://delicious.com +https://independent.co.uk +https://whatsapp.com +https://one.com +https://photobucket.com +https://ted.com +https://themeforest.net +https://homestead.com +https://google.ca +https://cnet.com +https://1und1.de +https://deviantart.com +https://scribd.com +https://jiathis.com +https://domainname.de +https://ca.gov +https://shopify.com +https://plesk.com +https://wiley.com +https://who.int +https://un.org +https://buzzfeed.com +https://theatlantic.com +https://barnesandnoble.com +https://sakura.ne.jp +https://pbs.org +https://nationalgeographic.com +https://getpocket.com +https://blogspot.com.es +https://nature.com +https://networksolutions.com +https://webmd.com +https://foxnews.com +https://cbsnews.com +https://techcrunch.com +https://booking.com +https://php.net +https://berkeley.edu +https://cloudfront.net +https://sciencedirect.com +https://ibm.com +https://a8.net +https://163.com +https://nbcnews.com +https://skype.com +https://mashable.com +https://cornell.edu +https://naver.com +https://domainretailing.com +https://usda.gov +https://wp.me +https://4.cn +https://springer.com +https://whitehouse.gov +https://squarespace.com +https://phoca.cz +https://change.org +https://cbc.ca +https://ft.com +https://epa.gov +https://secureserver.net +https://enable-javascript.com +https://meetup.com +https://noaa.gov +https://cnbc.com +https://nps.gov +https://phpbb.com +https://wikia.com +https://usnews.com +https://google.nl +https://myshopify.com +https://mapquest.com +https://trustpilot.com +https://domainactive.co +https://uol.com.br +https://foursquare.com +https://ow.ly +https://google.com.br +https://telegram.me +https://sohu.com +https://loc.gov +https://economist.com +https://fda.gov +https://irs.gov +https://themegrill.com +https://wufoo.com +https://geocities.jp +https://bigcartel.com +https://livedoor.jp +https://chicagotribune.com +https://dribbble.com +https://hp.com +https://doi.org +https://prnewswire.com +https://ed.gov +https://ok.ru +https://newyorker.com +https://abc.net.au +https://bizjournals.com +https://slate.com +https://houzz.com +https://umblr.com +https://fb.com +https://vice.com +https://xinhuanet.com +https://engadget.com +https://nifty.com +https://t.me +https://marriott.com +https://clickbank.net +https://globo.com +https://list-manage1.com +https://histats.com +https://state.gov +https://cbslocal.com +https://unesco.org +https://google.com.au +https://umich.edu +https://hostnet.nl +https://google.pl +https://house.gov +https://youku.com +https://theverge.com +https://ocn.ne.jp +https://storify.com +https://sogou.com +https://goo.ne.jp +https://fortune.com +https://wunderground.com +https://aboutcookies.org +https://rs6.net +https://columbia.edu +https://namejet.com +https://gofundme.com +https://oracle.com +https://yale.edu +https://psychologytoday.com +https://ifeng.com +https://washington.edu +https://indiatimes.com +https://samsung.com +https://upenn.edu +https://athemes.com +https://1and1.com +https://studiopress.com +https://hilton.com +https://debian.org +https://wikihow.com +https://fbcdn.net +https://fb.me +https://senate.gov +https://fastcompany.com +https://mailchimp.com +https://alibaba.com +https://youronlinechoices.com +https://android.com +https://researchgate.net +https://ustream.tv +https://dedecms.com +https://zdnet.com +https://home.pl +https://exblog.jp +https://cryoutcreations.eu +https://entrepreneur.com +https://drupal.org +https://sagepub.com +https://tripadvisor.co.uk +https://businesswire.com +https://shinystat.com +https://umn.edu +https://jugem.jp +https://hbr.org +https://sciencemag.org +https://ftc.gov +https://1688.com +https://google.co.in +https://wisc.edu +https://ucla.edu +https://inc.com +https://psu.edu +https://loopia.se +https://visma.com +https://dreamhost.com +https://mijndomein.nl +https://loopia.com +https://ox.ac.uk +https://scientificamerican.com +https://utexas.edu +https://stackoverflow.com +https://1and1.fr +https://sedo.com +https://mozilla.com +https://worldbank.org +https://hubspot.com +https://census.gov +https://arstechnica.com +https://mysql.com +https://si.edu +https://allaboutcookies.org +https://usgs.gov +https://intel.com +https://amazon.fr +https://shop-pro.jp +https://tandfonline.com +https://aliyun.com +https://office.com +https://alexa.com +https://zendesk.com +https://nhk.or.jp +https://colorlib.com +https://accuweather.com +https://cisco.com +https://google.ru +https://cam.ac.uk +https://hibu.com +https://hollywoodreporter.com +https://admin.ch +https://example.com +https://hhs.gov +https://twitch.tv +https://networkadvertising.org +https://nyu.edu +https://teamviewer.com +https://nazwa.pl +https://variety.com +https://netflix.com +https://box.com +https://prestashop.com +https://bls.gov +https://bmj.com +https://uchicago.edu +https://wsimg.com +https://www.nhs.uk +https://eventbrite.co.uk +https://opensource.org +https://zenfolio.com +https://blogspot.jp +https://usc.edu +https://va.gov +https://cmu.edu +https://oecd.org +https://ieee.org +https://mlb.com +https://ename.com.cn +https://usa.gov +https://steampowered.com +https://google.ch +https://redcross.org +https://bund.de +https://thehill.com +https://dictionary.com +https://360.cn +https://hostgator.com +https://icann.org +https://dot.gov +https://adweek.com +https://fao.org +https://sun.com +https://iubenda.com +https://gesetze-im-internet.de +https://tmall.com +https://today.com +https://nginx.org +https://xiti.com +https://venturebeat.com +https://snapchat.com +https://ietf.org +https://symantec.com +https://mhlw.go.jp +https://duke.edu +https://japanpost.jp +https://giphy.com +https://netscape.com +https://justgiving.com +https://sec.gov +https://illinois.edu +https://att.com +https://squareup.com +https://aboutads.info +https://gpo.gov +https://tucowsdomains.com +https://domainnameshop.com +https://plos.org +https://elsevier.com +https://biomedcentral.com +https://reference.com +https://oup.com +https://ssa.gov +https://libsyn.com +https://windowsphone.com +https://ny.gov +https://bigcommerce.com +https://oreilly.com +https://domeneshop.no +https://googleapis.com +https://artisteer.com +https://arxiv.org +https://thenextweb.com +https://google.be +https://gotowebinar.com +https://deloitte.com +https://blackberry.com +https://w3schools.com +https://dol.gov +https://python.org +https://siteorigin.com +https://ewebdevelopment.com +https://moz.com +https://warnerbros.com +https://justice.gov +https://quantcast.com +https://dhs.gov +https://java.com +https://fcc.gov +https://congress.gov +https://g.co +https://playstation.com +https://iso.org +https://opencart.com +https://eff.org +https://ucl.ac.uk +https://moodle.org +https://web.de +https://unsplash.com +https://msdn.com +https://nist.gov +https://unicef.org +https://mlit.go.jp +https://canada.ca +https://bitbucket.org +https://azurewebsites.net +https://nginx.com +https://dmca.com +https://etracker.de +https://mynavi.jp +https://aarp.org +https://gartner.com +https://starwoodhotels.com +https://typeform.com +https://acm.org +https://sedoparking.com +https://ticketmaster.com +https:/ \ No newline at end of file diff --git a/examples/csv/import.urls.with.ids.csv b/examples/csv/import.urls.with.ids.csv new file mode 100644 index 0000000..e43bcbe --- /dev/null +++ b/examples/csv/import.urls.with.ids.csv @@ -0,0 +1,1002 @@ +2Cn5xp,https://abc.net.au,2018-06-11T22:59:03Z,,,, +2M3Y6z,https://mapquest.com,2018-06-11T22:59:03Z,,,, +2SG8Ny,https://office.com,2018-06-11T22:59:03Z,,,, +2Zty7p,https://aarp.org,2018-06-11T22:59:03Z,,,, +2gkry5,https://1688.com,2018-06-11T22:59:03Z,,,, +2uWtfh,https://constantcontact.com,2018-06-11T22:59:03Z,,,, +2wsvNQ,https://wufoo.com,2018-06-11T22:59:03Z,,,, +2wzL9T,https://slideshare.net,2018-06-11T22:59:03Z,,,, +34N4r8,https://dribbble.com,2018-06-11T22:59:03Z,,,, +3AT6Xn,https://livejournal.com,2018-06-11T22:59:03Z,,,, +3BuQws,https://bizjournals.com,2018-06-11T22:59:03Z,,,, +3EBTgA,https://free.fr,2018-06-11T22:59:03Z,,,, +3LwS7y,https://quantcast.com,2018-06-11T22:59:03Z,,,, +44S3Qb,https://booking.com,2018-06-11T22:59:03Z,,,, +47Fr9v,https://rambler.ru,2018-06-11T22:59:03Z,,,, +49cSgN,https://arstechnica.com,2018-06-11T22:59:03Z,,,, +4C4Ldp,https://ftc.gov,2018-06-11T22:59:03Z,,,, +4EcC2h,https://hbr.org,2018-06-11T22:59:03Z,,,, +4FzWg3,https://creativecommons.org,2018-06-11T22:59:03Z,,,, +4RGN9a,https://sun.com,2018-06-11T22:59:03Z,,,, +4bh3MV,https://epa.gov,2018-06-11T22:59:03Z,,,, +4p9paT,https://umblr.com,2018-06-11T22:59:03Z,,,, +4sABta,https://jiathis.com,2018-06-11T22:59:03Z,,,, +52k2zf,https://sourceforge.net,2018-06-11T22:59:03Z,,,, +5GTH5V,https://nyu.edu,2018-06-11T22:59:03Z,,,, +5TeUKe,https://today.com,2018-06-11T22:59:03Z,,,, +5VuWgK,https://hilton.com,2018-06-11T22:59:03Z,,,, +5cr6TD,https://blogger.com,2018-06-11T22:59:03Z,,,, +5dsHZv,https://nih.gov,2018-06-11T22:59:03Z,,,, +5fLCFN,https://utexas.edu,2018-06-11T22:59:03Z,,,, +5m9RWH,https://usc.edu,2018-06-11T22:59:03Z,,,, +5rBev3,https://medium.com,2018-06-11T22:59:03Z,,,, +5z49Gu,https://gotowebinar.com,2018-06-11T22:59:03Z,,,, +62s6Cf,https://bls.gov,2018-06-11T22:59:03Z,,,, +65TwkB,https://opencart.com,2018-06-11T22:59:03Z,,,, +66YaSw,https://t.co,2018-06-11T22:59:03Z,,,, +6C2p7f,https://venturebeat.com,2018-06-11T22:59:03Z,,,, +6PNYcy,https://fbcdn.net,2018-06-11T22:59:03Z,,,, +6nHck4,https://starwoodhotels.com,2018-06-11T22:59:03Z,,,, +6vDgpp,https://bbc.co.uk,2018-06-11T22:59:03Z,,,, +7DBnHW,https://usgs.gov,2018-06-11T22:59:03Z,,,, +7GQxna,https://gpo.gov,2018-06-11T22:59:03Z,,,, +7WFUKF,https://xing.com,2018-06-11T22:59:03Z,,,, +7amge2,https://networkadvertising.org,2018-06-11T22:59:03Z,,,, +7tHLYf,https://box.com,2018-06-11T22:59:03Z,,,, +7vxgsd,https://disqus.com,2018-06-11T22:59:03Z,,,, +82zCJe,https://aol.com,2018-06-11T22:59:03Z,,,, +89zUtD,https://archive.org,2018-06-11T22:59:03Z,,,, +8DPRbx,https://line.me,2018-06-11T22:59:03Z,,,, +8LBWe8,https://eventbrite.co.uk,2018-06-11T22:59:03Z,,,, +8QFUk2,https://themeforest.net,2018-06-11T22:59:03Z,,,, +8YKBuv,https://oreilly.com,2018-06-11T22:59:03Z,,,, +8fMu6Y,https://dot.gov,2018-06-11T22:59:03Z,,,, +8wUVpK,https://mozilla.com,2018-06-11T22:59:03Z,,,, +926bk2,https://studiopress.com,2018-06-11T22:59:03Z,,,, +95Lre4,https://webs.com,2018-06-11T22:59:03Z,,,, +9Sn8wJ,https://google.pl,2018-06-11T22:59:03Z,,,, +9XaRmb,https://example.com,2018-06-11T22:59:03Z,,,, +9YESM2,https://google.co.uk,2018-06-11T22:59:03Z,,,, +9Yw3p3,https://globo.com,2018-06-11T22:59:03Z,,,, +9YwTQw,https://google.ru,2018-06-11T22:59:03Z,,,, +9cwX6F,https://secureserver.net,2018-06-11T22:59:03Z,,,, +9f2b9r,https://themegrill.com,2018-06-11T22:59:03Z,,,, +9mvbgu,https://athemes.com,2018-06-11T22:59:03Z,,,, +9pdNfg,https://github.io,2018-06-11T22:59:03Z,,,, +9wP2Zp,https://sedo.com,2018-06-11T22:59:03Z,,,, +A2BNY8,https://playstation.com,2018-06-11T22:59:03Z,,,, +AAtJbD,https://domainname.de,2018-06-11T22:59:03Z,,,, +ARJquH,https://artisteer.com,2018-06-11T22:59:03Z,,,, +ASVZkf,https://histats.com,2018-06-11T22:59:03Z,,,, +AeywWU,https://etracker.de,2018-06-11T22:59:03Z,,,, +AhAbNy,https://fastcompany.com,2018-06-11T22:59:03Z,,,, +An9sTx,https://android.com,2018-06-11T22:59:03Z,,,, +ArCdfR,https://mailchimp.com,2018-06-11T22:59:03Z,,,, +AubtB4,https://aboutcookies.org,2018-06-11T22:59:03Z,,,, +Ay9ZmZ,https://dailymail.co.uk,2018-06-11T22:59:03Z,,,, +AzQJ7f,https://list-manage.com,2018-06-11T22:59:03Z,,,, +B7fNrb,https://entrepreneur.com,2018-06-11T22:59:03Z,,,, +BA3CBu,https://blogspot.jp,2018-06-11T22:59:03Z,,,, +BFKpKz,https://flickr.com,2018-06-11T22:59:03Z,,,, +BL7YUK,https://1and1.fr,2018-06-11T22:59:03Z,,,, +BSntUf,https://linkedin.com,2018-06-11T22:59:03Z,,,, +BWrkQe,https://bund.de,2018-06-11T22:59:03Z,,,, +BYfWaR,https://elegantthemes.com,2018-06-11T22:59:03Z,,,, +BbMAkx,https://va.gov,2018-06-11T22:59:03Z,,,, +BqEKeb,https://getpocket.com,2018-06-11T22:59:03Z,,,, +C8dwVQ,https://hostgator.com,2018-06-11T22:59:03Z,,,, +CBY5Tt,https://hp.com,2018-06-11T22:59:03Z,,,, +CJBWQQ,https://joomla.org,2018-06-11T22:59:03Z,,,, +CQw48L,https://intel.com,2018-06-11T22:59:03Z,,,, +CRqt4Y,https://moz.com,2018-06-11T22:59:03Z,,,, +CVstJQ,https://wikimedia.org,2018-06-11T22:59:03Z,,,, +CknNa9,https://stackoverflow.com,2018-06-11T22:59:03Z,,,, +CnDRRf,https://ed.gov,2018-06-11T22:59:03Z,,,, +Cur9YQ,https://addtoany.com,2018-06-11T22:59:03Z,,,, +CwdR5u,https://dreamhost.com,2018-06-11T22:59:03Z,,,, +D54cHf,https://typeform.com,2018-06-11T22:59:03Z,,,, +D9vdkS,https://mit.edu,2018-06-11T22:59:03Z,,,, +DYFPqv,https://googleusercontent.com,2018-06-11T22:59:03Z,,,, +DZeQLT,https://theguardian.com,2018-06-11T22:59:03Z,,,, +DZy7HN,https://shopify.com,2018-06-11T22:59:03Z,,,, +Dk3eZc,https://drupal.org,2018-06-11T22:59:03Z,,,, +DnkD8V,https://duke.edu,2018-06-11T22:59:03Z,,,, +DwbCbq,https://telegram.me,2018-06-11T22:59:03Z,,,, +DzbVF6,https://dictionary.com,2018-06-11T22:59:03Z,,,, +E3Z5vN,https://dropbox.com,2018-06-11T22:59:03Z,,,, +E5ppFx,https://debian.org,2018-06-11T22:59:03Z,,,, +E77h2c,https://springer.com,2018-06-11T22:59:03Z,,,, +EA5cDy,https://about.com,2018-06-11T22:59:03Z,,,, +EAKzdV,https://symantec.com,2018-06-11T22:59:03Z,,,, +EAb3vZ,https://theverge.com,2018-06-11T22:59:03Z,,,, +ECpAm7,https://wordpress.com,2018-06-11T22:59:03Z,,,, +EJEYLx,https://forbes.com,2018-06-11T22:59:03Z,,,, +EMmLVA,https://ca.gov,2018-06-11T22:59:03Z,,,, +Egxr3k,https://gnu.org,2018-06-11T22:59:03Z,,,, +ExTgnF,https://mozilla.org,2018-06-11T22:59:03Z,,,, +F4QpLw,https://cornell.edu,2018-06-11T22:59:03Z,,,, +F7HqtQ,https://typepad.com,2018-06-11T22:59:03Z,,,, +FFaMrB,https://w3.org,2018-06-11T22:59:03Z,,,, +FP5JEb,https://sciencemag.org,2018-06-11T22:59:03Z,,,, +FgRKzW,https://squareup.com,2018-06-11T22:59:03Z,,,, +Fh4zbQ,https://unsplash.com,2018-06-11T22:59:03Z,,,, +FrbUaX,https://giphy.com,2018-06-11T22:59:03Z,,,, +FuRbbQ,https://vimeo.com,2018-06-11T22:59:03Z,,,, +G28FFZ,https://soundcloud.com,2018-06-11T22:59:03Z,,,, +GCqwFU,https://doi.org,2018-06-11T22:59:03Z,,,, +GDsuc8,https://networksolutions.com,2018-06-11T22:59:03Z,,,, +GNmBrr,https://vk.com,2018-06-11T22:59:03Z,,,, +GR2BDU,https://ft.com,2018-06-11T22:59:03Z,,,, +Gam5Fb,https://alexa.com,2018-06-11T22:59:03Z,,,, +Gh6cv4,https://usda.gov,2018-06-11T22:59:03Z,,,, +H5zqYC,https://buzzfeed.com,2018-06-11T22:59:03Z,,,, +HLqesU,https://google.de,2018-06-11T22:59:03Z,,,, +HMhEXJ,https://trustpilot.com,2018-06-11T22:59:03Z,,,, +HN2eYZ,https://nginx.com,2018-06-11T22:59:03Z,,,, +HRr37m,https://icio.us,2018-06-11T22:59:03Z,,,, +HSRruR,https://congress.gov,2018-06-11T22:59:03Z,,,, +HnDQFV,https://barnesandnoble.com,2018-06-11T22:59:03Z,,,, +J33BBN,https://wp.com,2018-06-11T22:59:03Z,,,, +JAtXNr,https://ucl.ac.uk,2018-06-11T22:59:03Z,,,, +JCAZYJ,https://ow.ly,2018-06-11T22:59:03Z,,,, +JEm5av,https://latimes.com,2018-06-11T22:59:03Z,,,, +JFeL8f,https://youtu.be,2018-06-11T22:59:03Z,,,, +JJgZ6z,https://bing.com,2018-06-11T22:59:03Z,,,, +JdVEXg,https://sina.com.cn,2018-06-11T22:59:03Z,,,, +JgCAdR,https://photobucket.com,2018-06-11T22:59:03Z,,,, +Jtzmhh,https://techcrunch.com,2018-06-11T22:59:03Z,,,, +JuHtZG,https://google.it,2018-06-11T22:59:03Z,,,, +K7XkAZ,https://census.gov,2018-06-11T22:59:03Z,,,, +KJUJbU,https://dhs.gov,2018-06-11T22:59:03Z,,,, +KNtT3Q,https://bitbucket.org,2018-06-11T22:59:03Z,,,, +Km2DaY,https://fortune.com,2018-06-11T22:59:03Z,,,, +Krqaq3,https://samsung.com,2018-06-11T22:59:03Z,,,, +KvGxLq,https://ticketmaster.com,2018-06-11T22:59:03Z,,,, +L6agWc,https://google.nl,2018-06-11T22:59:03Z,,,, +L8N5AN,https://foxnews.com,2018-06-11T22:59:03Z,,,, +LAAGMv,https://issuu.com,2018-06-11T22:59:03Z,,,, +LD8WuS,https://wiley.com,2018-06-11T22:59:03Z,,,, +LFq9rZ,https://dol.gov,2018-06-11T22:59:03Z,,,, +LM3r7a,https://opera.com,2018-06-11T22:59:03Z,,,, +LWyaVp,https://blackberry.com,2018-06-11T22:59:03Z,,,, +LXEEFY,https://nazwa.pl,2018-06-11T22:59:03Z,,,, +Lhk4Eh,https://xinhuanet.com,2018-06-11T22:59:03Z,,,, +LhyHBM,https://windowsphone.com,2018-06-11T22:59:03Z,,,, +LkHTrN,https://jimdo.com,2018-06-11T22:59:03Z,,,, +MBSpmv,https://google.fr,2018-06-11T22:59:03Z,,,, +MQbUhe,https://parallels.com,2018-06-11T22:59:03Z,,,, +MSwUbC,https://acm.org,2018-06-11T22:59:03Z,,,, +MU2g8X,https://redcross.org,2018-06-11T22:59:03Z,,,, +MVF33U,https://sohu.com,2018-06-11T22:59:03Z,,,, +MYKfHG,https://state.gov,2018-06-11T22:59:03Z,,,, +MYt99N,https://thehill.com,2018-06-11T22:59:03Z,,,, +MhV65H,https://canada.ca,2018-06-11T22:59:03Z,,,, +N4Ubvt,https://facebook.com,2018-06-11T22:59:03Z,,,, +N7QDJs,https://att.com,2018-06-11T22:59:03Z,,,, +ND4TtB,https://arxiv.org,2018-06-11T22:59:03Z,,,, +NGdchr,https://wisc.edu,2018-06-11T22:59:03Z,,,, +NPMEVq,https://washington.edu,2018-06-11T22:59:03Z,,,, +NQ8yNz,https://prnewswire.com,2018-06-11T22:59:03Z,,,, +NSJJny,https://sagepub.com,2018-06-11T22:59:03Z,,,, +NYaMpe,https://www.nhs.uk,2018-06-11T22:59:03Z,,,, +NhzfA5,https://ieee.org,2018-06-11T22:59:03Z,,,, +Nkd6Q3,https://oracle.com,2018-06-11T22:59:03Z,,,, +NyMEGK,https:/,2018-06-11T22:59:03Z,,,, +P59Yd3,https://nginx.org,2018-06-11T22:59:03Z,,,, +P7WXNu,https://mijndomein.nl,2018-06-11T22:59:03Z,,,, +PFh3L2,https://cpanel.com,2018-06-11T22:59:03Z,,,, +PL9mLN,https://miibeian.gov.cn,2018-06-11T22:59:03Z,,,, +PX79hx,https://worldbank.org,2018-06-11T22:59:03Z,,,, +PY69W4,https://cdc.gov,2018-06-11T22:59:03Z,,,, +PfYRVH,https://paypal.com,2018-06-11T22:59:03Z,,,, +PgF38a,https://www.gov.uk,2018-06-11T22:59:03Z,,,, +PkDtSt,https://meetup.com,2018-06-11T22:59:03Z,,,, +Puv4C5,https://4.cn,2018-06-11T22:59:03Z,,,, +Q2nZE4,https://cnbc.com,2018-06-11T22:59:03Z,,,, +Q4fxRd,https://colorlib.com,2018-06-11T22:59:03Z,,,, +QAqWZF,https://godaddy.com,2018-06-11T22:59:03Z,,,, +QDVD6n,https://weibo.com,2018-06-11T22:59:03Z,,,, +QFB4GG,https://nps.gov,2018-06-11T22:59:03Z,,,, +QGcEBd,https://icann.org,2018-06-11T22:59:03Z,,,, +QMW2aA,https://yandex.ru,2018-06-11T22:59:03Z,,,, +QRVA4e,https://miitbeian.gov.cn,2018-06-11T22:59:03Z,,,, +QaLgNw,https://sogou.com,2018-06-11T22:59:03Z,,,, +Qge2FM,https://aboutads.info,2018-06-11T22:59:03Z,,,, +QzXaJf,https://ename.com.cn,2018-06-11T22:59:03Z,,,, +Qzat7E,https://jugem.jp,2018-06-11T22:59:03Z,,,, +R39hnB,https://un.org,2018-06-11T22:59:03Z,,,, +R7kX3b,https://tandfonline.com,2018-06-11T22:59:03Z,,,, +RB4tm7,https://psu.edu,2018-06-11T22:59:03Z,,,, +RV7Ztb,https://azurewebsites.net,2018-06-11T22:59:03Z,,,, +RdKtms,https://justice.gov,2018-06-11T22:59:03Z,,,, +RhTvPX,https://cisco.com,2018-06-11T22:59:03Z,,,, +RuCCwJ,https://twitter.com,2018-06-11T22:59:03Z,,,, +RugBuh,https://siteorigin.com,2018-06-11T22:59:03Z,,,, +S5WJ9G,https://etsy.com,2018-06-11T22:59:03Z,,,, +S8LD4u,https://nhk.or.jp,2018-06-11T22:59:03Z,,,, +SHR65R,https://newyorker.com,2018-06-11T22:59:03Z,,,, +SQPV7Y,https://cbc.ca,2018-06-11T22:59:03Z,,,, +SUX2p9,https://foursquare.com,2018-06-11T22:59:03Z,,,, +SUZHuZ,https://mysql.com,2018-06-11T22:59:03Z,,,, +SZYUhQ,https://slate.com,2018-06-11T22:59:03Z,,,, +Senk2m,https://inc.com,2018-06-11T22:59:03Z,,,, +Stc2u2,https://php.net,2018-06-11T22:59:03Z,,,, +SunWsf,https://1und1.de,2018-06-11T22:59:03Z,,,, +SwQRET,https://wunderground.com,2018-06-11T22:59:03Z,,,, +TGFRGh,https://bmj.com,2018-06-11T22:59:03Z,,,, +TYgcPD,https://xiti.com,2018-06-11T22:59:03Z,,,, +TaRf9k,https://tumblr.com,2018-06-11T22:59:03Z,,,, +TgP2pT,https://shop-pro.jp,2018-06-11T22:59:03Z,,,, +Ts77ba,https://ssa.gov,2018-06-11T22:59:03Z,,,, +TuKV7f,https://bbc.com,2018-06-11T22:59:03Z,,,, +UAygVS,https://skype.com,2018-06-11T22:59:03Z,,,, +UBPfxn,https://umich.edu,2018-06-11T22:59:03Z,,,, +UEMNwV,https://ameblo.jp,2018-06-11T22:59:03Z,,,, +UHFZbA,https://cloudfront.net,2018-06-11T22:59:03Z,,,, +UQEHch,https://oecd.org,2018-06-11T22:59:03Z,,,, +UQddnp,https://reuters.com,2018-06-11T22:59:03Z,,,, +UXCGYt,https://twitch.tv,2018-06-11T22:59:03Z,,,, +UXkfGe,https://wired.com,2018-06-11T22:59:03Z,,,, +UuA2w6,https://gravatar.com,2018-06-11T22:59:03Z,,,, +UyNqkV,https://amazon.fr,2018-06-11T22:59:03Z,,,, +V6gKJL,https://myspace.com,2018-06-11T22:59:03Z,,,, +VFKhnw,https://exblog.jp,2018-06-11T22:59:03Z,,,, +VGwBzH,https://g.co,2018-06-11T22:59:03Z,,,, +VLGKWe,https://ifeng.com,2018-06-11T22:59:03Z,,,, +VRtDKK,https://nifty.com,2018-06-11T22:59:03Z,,,, +VSxuvS,https://ebay.com,2018-06-11T22:59:03Z,,,, +VWSYB4,https://houzz.com,2018-06-11T22:59:03Z,,,, +VbEb9p,https://statcounter.com,2018-06-11T22:59:03Z,,,, +VbRFEs,https://ny.gov,2018-06-11T22:59:03Z,,,, +Vc7yAN,https://shinystat.com,2018-06-11T22:59:03Z,,,, +VfEJzb,https://google.ca,2018-06-11T22:59:03Z,,,, +VhntJs,https://steampowered.com,2018-06-11T22:59:03Z,,,, +VpsE2B,https://fao.org,2018-06-11T22:59:03Z,,,, +VqYNzZ,https://loc.gov,2018-06-11T22:59:03Z,,,, +VvMzH5,https://tripadvisor.co.uk,2018-06-11T22:59:03Z,,,, +Vwvfav,https://unicef.org,2018-06-11T22:59:03Z,,,, +W2rHJy,https://domainnameshop.com,2018-06-11T22:59:03Z,,,, +WH7nFv,https://usnews.com,2018-06-11T22:59:03Z,,,, +WaX9pw,https://nature.com,2018-06-11T22:59:03Z,,,, +WbdpXU,https://cbsnews.com,2018-06-11T22:59:03Z,,,, +Wmppmv,https://yale.edu,2018-06-11T22:59:03Z,,,, +Wyxxtz,https://domainactive.co,2018-06-11T22:59:03Z,,,, +XQfHfR,https://usatoday.com,2018-06-11T22:59:03Z,,,, +XTyNrB,https://accuweather.com,2018-06-11T22:59:03Z,,,, +XUmGuX,https://netflix.com,2018-06-11T22:59:03Z,,,, +XXhWHV,https://instagram.com,2018-06-11T22:59:03Z,,,, +XdN7zZ,https://youronlinechoices.com,2018-06-11T22:59:03Z,,,, +XuXr42,https://blogspot.com,2018-06-11T22:59:03Z,,,, +YCxcKX,https://scribd.com,2018-06-11T22:59:03Z,,,, +YHFG3t,https://tmall.com,2018-06-11T22:59:03Z,,,, +YQcRDE,https://whitehouse.gov,2018-06-11T22:59:03Z,,,, +YSGrua,https://deloitte.com,2018-06-11T22:59:03Z,,,, +YW25am,https://cryoutcreations.eu,2018-06-11T22:59:03Z,,,, +YWDXcC,https://researchgate.net,2018-06-11T22:59:03Z,,,, +YfGT5s,https://clickbank.net,2018-06-11T22:59:03Z,,,, +YgxZHZ,https://live.com,2018-06-11T22:59:03Z,,,, +YnQSB4,https://sedoparking.com,2018-06-11T22:59:03Z,,,, +Yr8tkb,https://gartner.com,2018-06-11T22:59:03Z,,,, +YzrmTT,https://google.com.br,2018-06-11T22:59:03Z,,,, +Z43cwq,https://sciencedirect.com,2018-06-11T22:59:03Z,,,, +Z63ATa,https://admin.ch,2018-06-11T22:59:03Z,,,, +ZV72RB,https://homestead.com,2018-06-11T22:59:03Z,,,, +ZfFUg9,https://amazon.co.uk,2018-06-11T22:59:03Z,,,, +ZgtdKE,https://beian.gov.cn,2018-06-11T22:59:03Z,,,, +Zk3Q25,https://tripadvisor.com,2018-06-11T22:59:03Z,,,, +ZneUUv,https://oup.com,2018-06-11T22:59:03Z,,,, +ZucJHG,https://geocities.jp,2018-06-11T22:59:03Z,,,, +a4S54H,https://zdnet.com,2018-06-11T22:59:03Z,,,, +aJLPzU,https://imgur.com,2018-06-11T22:59:03Z,,,, +aKVBnt,https://telegraph.co.uk,2018-06-11T22:59:03Z,,,, +apBwhz,https://google.com,2018-06-11T22:59:03Z,,,, +as8GCJ,https://gesetze-im-internet.de,2018-06-11T22:59:03Z,,,, +avWzXs,https://mlb.com,2018-06-11T22:59:03Z,,,, +b4Txyh,https://hollywoodreporter.com,2018-06-11T22:59:03Z,,,, +b7xy9X,https://myshopify.com,2018-06-11T22:59:03Z,,,, +bCvPgq,https://zendesk.com,2018-06-11T22:59:03Z,,,, +bXMdSb,https://marriott.com,2018-06-11T22:59:03Z,,,, +bksxzV,https://e-recht24.de,2018-06-11T22:59:03Z,,,, +bkwApT,https://gofundme.com,2018-06-11T22:59:03Z,,,, +brxd9r,https://wp.me,2018-06-11T22:59:03Z,,,, +cEWRu2,https://iubenda.com,2018-06-11T22:59:03Z,,,, +ccuBmQ,https://weebly.com,2018-06-11T22:59:03Z,,,, +cgzUc4,https://googleapis.com,2018-06-11T22:59:03Z,,,, +chmqTm,https://ox.ac.uk,2018-06-11T22:59:03Z,,,, +cmy4Fs,https://engadget.com,2018-06-11T22:59:03Z,,,, +cn4fn9,https://360.cn,2018-06-11T22:59:03Z,,,, +cssxWM,https://berkeley.edu,2018-06-11T22:59:03Z,,,, +cuB2ww,https://uchicago.edu,2018-06-11T22:59:03Z,,,, +d2WBFd,https://reference.com,2018-06-11T22:59:03Z,,,, +d4v9xP,https://python.org,2018-06-11T22:59:03Z,,,, +d68ksJ,https://wordpress.org,2018-06-11T22:59:03Z,,,, +dMpLg5,https://livedoor.jp,2018-06-11T22:59:03Z,,,, +dMwpeZ,https://variety.com,2018-06-11T22:59:03Z,,,, +dS4zUK,https://imdb.com,2018-06-11T22:59:03Z,,,, +dUapWs,https://prestashop.com,2018-06-11T22:59:03Z,,,, +dcJJ2t,https://cpanel.net,2018-06-11T22:59:03Z,,,, +dnRCCf,https://w3schools.com,2018-06-11T22:59:03Z,,,, +dqwz8D,https://doubleclick.net,2018-06-11T22:59:03Z,,,, +dw8mSF,https://huffingtonpost.com,2018-06-11T22:59:03Z,,,, +eFpX3J,https://pinterest.com,2018-06-11T22:59:03Z,,,, +eMJ4H8,https://nbcnews.com,2018-06-11T22:59:03Z,,,, +eYgbcq,https://fc2.com,2018-06-11T22:59:03Z,,,, +eaL7aJ,https://libsyn.com,2018-06-11T22:59:03Z,,,, +eauUpr,https://ocn.ne.jp,2018-06-11T22:59:03Z,,,, +ekGPe9,https://illinois.edu,2018-06-11T22:59:03Z,,,, +emcsUH,https://youtube.com,2018-06-11T22:59:03Z,,,, +f224CV,https://squarespace.com,2018-06-11T22:59:03Z,,,, +f7yM3D,https://dmca.com,2018-06-11T22:59:03Z,,,, +fMX49B,https://wsj.com,2018-06-11T22:59:03Z,,,, +fMpzev,https://wixsite.com,2018-06-11T22:59:03Z,,,, +faVVSh,https://phpbb.com,2018-06-11T22:59:03Z,,,, +fcTJAx,https://npr.org,2018-06-11T22:59:03Z,,,, +fcThcy,https://dedecms.com,2018-06-11T22:59:03Z,,,, +ft3sHd,https://adweek.com,2018-06-11T22:59:03Z,,,, +g2KsDb,https://stumbleupon.com,2018-06-11T22:59:03Z,,,, +gD9mTP,https://apache.org,2018-06-11T22:59:03Z,,,, +gG6rSA,https://ucla.edu,2018-06-11T22:59:03Z,,,, +gGMVkM,https://amazon.com,2018-06-11T22:59:03Z,,,, +gHvRed,https://digg.com,2018-06-11T22:59:03Z,,,, +gMHZaU,https://youku.com,2018-06-11T22:59:03Z,,,, +gQqubm,https://naver.com,2018-06-11T22:59:03Z,,,, +gbaDPs,https://elsevier.com,2018-06-11T22:59:03Z,,,, +gdczmp,https://upenn.edu,2018-06-11T22:59:03Z,,,, +gfHSGr,https://independent.co.uk,2018-06-11T22:59:03Z,,,, +gp56zv,https://si.edu,2018-06-11T22:59:03Z,,,, +h6yr7K,https://wikihow.com,2018-06-11T22:59:03Z,,,, +hE8QMY,https://visma.com,2018-06-11T22:59:03Z,,,, +hEkZP4,https://home.pl,2018-06-11T22:59:03Z,,,, +hHJkgw,https://1and1.com,2018-06-11T22:59:03Z,,,, +hQUFE7,https://yahoo.com,2018-06-11T22:59:03Z,,,, +hUt7xe,https://kickstarter.com,2018-06-11T22:59:03Z,,,, +hWMwXp,https://cbslocal.com,2018-06-11T22:59:03Z,,,, +haDer4,https://loopia.com,2018-06-11T22:59:03Z,,,, +hnF8LQ,https://baidu.com,2018-06-11T22:59:03Z,,,, +https:/,NyMEGK,,,,, +https://163.com,ptN9pf,,,,, +https://1688.com,2gkry5,,,,, +https://1and1.com,hHJkgw,,,,, +https://1and1.fr,BL7YUK,,,,, +https://1und1.de,SunWsf,,,,, +https://360.cn,cn4fn9,,,,, +https://4.cn,Puv4C5,,,,, +https://51.la,wyUCAe,,,,, +https://a8.net,m57dUp,,,,, +https://aarp.org,2Zty7p,,,,, +https://abc.net.au,2Cn5xp,,,,, +https://about.com,EA5cDy,,,,, +https://aboutads.info,Qge2FM,,,,, +https://aboutcookies.org,AubtB4,,,,, +https://accuweather.com,XTyNrB,,,,, +https://acm.org,MSwUbC,,,,, +https://addthis.com,yBNq4m,,,,, +https://addtoany.com,Cur9YQ,,,,, +https://admin.ch,Z63ATa,,,,, +https://adobe.com,pJyTHb,,,,, +https://adweek.com,ft3sHd,,,,, +https://alexa.com,Gam5Fb,,,,, +https://alibaba.com,q7zbJ5,,,,, +https://aliyun.com,zyPwNq,,,,, +https://allaboutcookies.org,ySvWBH,,,,, +https://amazon.co.jp,ufdmPd,,,,, +https://amazon.co.uk,ZfFUg9,,,,, +https://amazon.com,gGMVkM,,,,, +https://amazon.de,qr4CgU,,,,, +https://amazon.fr,UyNqkV,,,,, +https://amazonaws.com,m8nEGs,,,,, +https://ameblo.jp,UEMNwV,,,,, +https://amzn.to,k99Re9,,,,, +https://android.com,An9sTx,,,,, +https://aol.com,82zCJe,,,,, +https://apache.org,gD9mTP,,,,, +https://apple.com,wWkkv8,,,,, +https://archive.org,89zUtD,,,,, +https://arstechnica.com,49cSgN,,,,, +https://artisteer.com,ARJquH,,,,, +https://arxiv.org,ND4TtB,,,,, +https://athemes.com,9mvbgu,,,,, +https://att.com,N7QDJs,,,,, +https://azurewebsites.net,RV7Ztb,,,,, +https://baidu.com,hnF8LQ,,,,, +https://bandcamp.com,sLYcUG,,,,, +https://barnesandnoble.com,HnDQFV,,,,, +https://bbb.org,kDrgJL,,,,, +https://bbc.co.uk,6vDgpp,,,,, +https://bbc.com,TuKV7f,,,,, +https://behance.net,w4dDcB,,,,, +https://beian.gov.cn,ZgtdKE,,,,, +https://berkeley.edu,cssxWM,,,,, +https://bigcartel.com,wKxV53,,,,, +https://bigcommerce.com,un9Pkg,,,,, +https://bing.com,JJgZ6z,,,,, +https://biomedcentral.com,vEmdB8,,,,, +https://bit.ly,qv2R5m,,,,, +https://bitbucket.org,KNtT3Q,,,,, +https://bizjournals.com,3BuQws,,,,, +https://blackberry.com,LWyaVp,,,,, +https://blogger.com,5cr6TD,,,,, +https://blogspot.co.uk,vkaedE,,,,, +https://blogspot.com,XuXr42,,,,, +https://blogspot.com.es,sGNMLW,,,,, +https://blogspot.jp,BA3CBu,,,,, +https://bloomberg.com,u8z9xW,,,,, +https://bls.gov,62s6Cf,,,,, +https://bluehost.com,tkCTth,,,,, +https://bmj.com,TGFRGh,,,,, +https://booking.com,44S3Qb,,,,, +https://box.com,7tHLYf,,,,, +https://bund.de,BWrkQe,,,,, +https://businessinsider.com,zmnEUS,,,,, +https://businesswire.com,uWB3RB,,,,, +https://buydomains.com,kDzdzJ,,,,, +https://buzzfeed.com,H5zqYC,,,,, +https://ca.gov,EMmLVA,,,,, +https://cam.ac.uk,sEtApG,,,,, +https://canada.ca,MhV65H,,,,, +https://cbc.ca,SQPV7Y,,,,, +https://cbslocal.com,hWMwXp,,,,, +https://cbsnews.com,WbdpXU,,,,, +https://cdc.gov,PY69W4,,,,, +https://census.gov,K7XkAZ,,,,, +https://change.org,yY2DGh,,,,, +https://chicagotribune.com,krHE7C,,,,, +https://cisco.com,RhTvPX,,,,, +https://clickbank.net,YfGT5s,,,,, +https://cloudfront.net,UHFZbA,,,,, +https://cmu.edu,qs4EfZ,,,,, +https://cnbc.com,Q2nZE4,,,,, +https://cnet.com,p2mUXv,,,,, +https://cnn.com,vkFbaR,,,,, +https://colorlib.com,Q4fxRd,,,,, +https://columbia.edu,nJZpMb,,,,, +https://congress.gov,HSRruR,,,,, +https://constantcontact.com,2uWtfh,,,,, +https://cornell.edu,F4QpLw,,,,, +https://cpanel.com,PFh3L2,,,,, +https://cpanel.net,dcJJ2t,,,,, +https://creativecommons.org,4FzWg3,,,,, +https://cryoutcreations.eu,YW25am,,,,, +https://dailymail.co.uk,Ay9ZmZ,,,,, +https://dailymotion.com,z9ZHQQ,,,,, +https://debian.org,E5ppFx,,,,, +https://dedecms.com,fcThcy,,,,, +https://delicious.com,srddMa,,,,, +https://deloitte.com,YSGrua,,,,, +https://deviantart.com,uYWA7x,,,,, +https://dhs.gov,KJUJbU,,,,, +https://dictionary.com,DzbVF6,,,,, +https://digg.com,gHvRed,,,,, +https://disqus.com,7vxgsd,,,,, +https://dmca.com,f7yM3D,,,,, +https://doi.org,GCqwFU,,,,, +https://dol.gov,LFq9rZ,,,,, +https://domainactive.co,Wyxxtz,,,,, +https://domainname.de,AAtJbD,,,,, +https://domainnameshop.com,W2rHJy,,,,, +https://domainretailing.com,pXmq2C,,,,, +https://domeneshop.no,r8tcAG,,,,, +https://dot.gov,8fMu6Y,,,,, +https://doubleclick.net,dqwz8D,,,,, +https://dreamhost.com,CwdR5u,,,,, +https://dribbble.com,34N4r8,,,,, +https://dropbox.com,E3Z5vN,,,,, +https://drupal.org,Dk3eZc,,,,, +https://duke.edu,DnkD8V,,,,, +https://e-recht24.de,bksxzV,,,,, +https://ebay.com,VSxuvS,,,,, +https://economist.com,ysGShJ,,,,, +https://ed.gov,CnDRRf,,,,, +https://eepurl.com,zagnaG,,,,, +https://eff.org,pEzkmx,,,,, +https://elegantthemes.com,BYfWaR,,,,, +https://elsevier.com,gbaDPs,,,,, +https://enable-javascript.com,yDgRDy,,,,, +https://ename.com.cn,QzXaJf,,,,, +https://engadget.com,cmy4Fs,,,,, +https://entrepreneur.com,B7fNrb,,,,, +https://epa.gov,4bh3MV,,,,, +https://etracker.de,AeywWU,,,,, +https://etsy.com,S5WJ9G,,,,, +https://europa.eu,nht265,,,,, +https://eventbrite.co.uk,8LBWe8,,,,, +https://eventbrite.com,sBZ2am,,,,, +https://ewebdevelopment.com,rWuJwU,,,,, +https://example.com,9XaRmb,,,,, +https://exblog.jp,VFKhnw,,,,, +https://facebook.com,N4Ubvt,,,,, +https://fao.org,VpsE2B,,,,, +https://fastcompany.com,AhAbNy,,,,, +https://fb.com,xNzmwV,,,,, +https://fb.me,tYcFwg,,,,, +https://fbcdn.net,6PNYcy,,,,, +https://fc2.com,eYgbcq,,,,, +https://fcc.gov,xsdVgM,,,,, +https://fda.gov,rUSgBk,,,,, +https://feedburner.com,sBDKBC,,,,, +https://flickr.com,BFKpKz,,,,, +https://forbes.com,EJEYLx,,,,, +https://fortune.com,Km2DaY,,,,, +https://foursquare.com,SUX2p9,,,,, +https://foxnews.com,L8N5AN,,,,, +https://free.fr,3EBTgA,,,,, +https://ft.com,GR2BDU,,,,, +https://ftc.gov,4C4Ldp,,,,, +https://g.co,VGwBzH,,,,, +https://gartner.com,Yr8tkb,,,,, +https://geocities.jp,ZucJHG,,,,, +https://gesetze-im-internet.de,as8GCJ,,,,, +https://getpocket.com,BqEKeb,,,,, +https://giphy.com,FrbUaX,,,,, +https://github.com,yFxM7M,,,,, +https://github.io,9pdNfg,,,,, +https://globo.com,9Yw3p3,,,,, +https://gnu.org,Egxr3k,,,,, +https://go.com,sHxY2E,,,,, +https://godaddy.com,QAqWZF,,,,, +https://gofundme.com,bkwApT,,,,, +https://goo.gl,qnD4XJ,,,,, +https://goo.ne.jp,qVygK6,,,,, +https://goodreads.com,x9EbEB,,,,, +https://google.be,te7adS,,,,, +https://google.ca,VfEJzb,,,,, +https://google.ch,vtWG9W,,,,, +https://google.co.in,tM2RBF,,,,, +https://google.co.jp,yMShaY,,,,, +https://google.co.uk,9YESM2,,,,, +https://google.com,apBwhz,,,,, +https://google.com.au,yMFySb,,,,, +https://google.com.br,YzrmTT,,,,, +https://google.de,HLqesU,,,,, +https://google.es,vEu4Th,,,,, +https://google.fr,MBSpmv,,,,, +https://google.it,JuHtZG,,,,, +https://google.nl,L6agWc,,,,, +https://google.pl,9Sn8wJ,,,,, +https://google.ru,9YwTQw,,,,, +https://googleapis.com,cgzUc4,,,,, +https://googleusercontent.com,DYFPqv,,,,, +https://gotowebinar.com,5z49Gu,,,,, +https://gpo.gov,7GQxna,,,,, +https://gravatar.com,UuA2w6,,,,, +https://guardian.co.uk,u2T9RE,,,,, +https://harvard.edu,muKXKA,,,,, +https://hatena.ne.jp,xUBfwT,,,,, +https://hbr.org,4EcC2h,,,,, +https://hhs.gov,sPGEkh,,,,, +https://hibu.com,sSzEZm,,,,, +https://hilton.com,5VuWgK,,,,, +https://histats.com,ASVZkf,,,,, +https://hollywoodreporter.com,b4Txyh,,,,, +https://home.pl,hEkZP4,,,,, +https://homestead.com,ZV72RB,,,,, +https://hostgator.com,C8dwVQ,,,,, +https://hostnet.nl,kS6XCE,,,,, +https://house.gov,qyCWYV,,,,, +https://houzz.com,VWSYB4,,,,, +https://hp.com,CBY5Tt,,,,, +https://hubspot.com,vsbRsK,,,,, +https://huffingtonpost.com,dw8mSF,,,,, +https://ibm.com,zYu9LW,,,,, +https://icann.org,QGcEBd,,,,, +https://icio.us,HRr37m,,,,, +https://ieee.org,NhzfA5,,,,, +https://ietf.org,sVCfW8,,,,, +https://ifeng.com,VLGKWe,,,,, +https://illinois.edu,ekGPe9,,,,, +https://imdb.com,dS4zUK,,,,, +https://imgur.com,aJLPzU,,,,, +https://inc.com,Senk2m,,,,, +https://independent.co.uk,gfHSGr,,,,, +https://indiatimes.com,qh24JD,,,,, +https://instagram.com,XXhWHV,,,,, +https://intel.com,CQw48L,,,,, +https://irs.gov,t8F4qH,,,,, +https://iso.org,zaJPJ2,,,,, +https://issuu.com,LAAGMv,,,,, +https://iubenda.com,cEWRu2,,,,, +https://japanpost.jp,xcL2KB,,,,, +https://java.com,qqzgpC,,,,, +https://jiathis.com,4sABta,,,,, +https://jimdo.com,LkHTrN,,,,, +https://joomla.org,CJBWQQ,,,,, +https://jugem.jp,Qzat7E,,,,, +https://justgiving.com,tJBpkh,,,,, +https://justice.gov,RdKtms,,,,, +https://kickstarter.com,hUt7xe,,,,, +https://latimes.com,JEm5av,,,,, +https://libsyn.com,eaL7aJ,,,,, +https://line.me,8DPRbx,,,,, +https://linkedin.com,BSntUf,,,,, +https://list-manage.com,AzQJ7f,,,,, +https://list-manage1.com,vzkZh7,,,,, +https://live.com,YgxZHZ,,,,, +https://livedoor.jp,dMpLg5,,,,, +https://livejournal.com,3AT6Xn,,,,, +https://loc.gov,VqYNzZ,,,,, +https://loopia.com,haDer4,,,,, +https://loopia.se,umkWBr,,,,, +https://macromedia.com,ppRLR5,,,,, +https://mail.ru,k2CqF3,,,,, +https://mailchimp.com,ArCdfR,,,,, +https://mapquest.com,2M3Y6z,,,,, +https://marriott.com,bXMdSb,,,,, +https://mashable.com,pWWnAF,,,,, +https://medium.com,5rBev3,,,,, +https://meetup.com,PkDtSt,,,,, +https://mhlw.go.jp,xJ69AD,,,,, +https://microsoft.com,tHPaHp,,,,, +https://miibeian.gov.cn,PL9mLN,,,,, +https://miitbeian.gov.cn,QRVA4e,,,,, +https://mijndomein.nl,P7WXNu,,,,, +https://mit.edu,D9vdkS,,,,, +https://mlb.com,avWzXs,,,,, +https://mlit.go.jp,yDVbFL,,,,, +https://moodle.org,nVgcGL,,,,, +https://moz.com,CRqt4Y,,,,, +https://mozilla.com,8wUVpK,,,,, +https://mozilla.org,ExTgnF,,,,, +https://msdn.com,t6FKQa,,,,, +https://msn.com,wF2uVC,,,,, +https://mynavi.jp,n6uFCc,,,,, +https://myshopify.com,b7xy9X,,,,, +https://myspace.com,V6gKJL,,,,, +https://mysql.com,SUZHuZ,,,,, +https://namejet.com,qn6aVp,,,,, +https://nasa.gov,ugKHdK,,,,, +https://nationalgeographic.com,wHCbXq,,,,, +https://nature.com,WaX9pw,,,,, +https://naver.com,gQqubm,,,,, +https://nazwa.pl,LXEEFY,,,,, +https://nbcnews.com,eMJ4H8,,,,, +https://netflix.com,XUmGuX,,,,, +https://netscape.com,uBpHuF,,,,, +https://networkadvertising.org,7amge2,,,,, +https://networksolutions.com,GDsuc8,,,,, +https://newyorker.com,SHR65R,,,,, +https://nginx.com,HN2eYZ,,,,, +https://nginx.org,P59Yd3,,,,, +https://nhk.or.jp,S8LD4u,,,,, +https://nifty.com,VRtDKK,,,,, +https://nih.gov,5dsHZv,,,,, +https://nist.gov,mb78SA,,,,, +https://noaa.gov,hxHYLL,,,,, +https://npr.org,fcTJAx,,,,, +https://nps.gov,QFB4GG,,,,, +https://ny.gov,VbRFEs,,,,, +https://nytimes.com,yqqECD,,,,, +https://nyu.edu,5GTH5V,,,,, +https://ocn.ne.jp,eauUpr,,,,, +https://oecd.org,UQEHch,,,,, +https://office.com,2SG8Ny,,,,, +https://ok.ru,uR3PMX,,,,, +https://one.com,qJMJZy,,,,, +https://opencart.com,65TwkB,,,,, +https://opensource.org,s8h7y4,,,,, +https://opera.com,LM3r7a,,,,, +https://oracle.com,Nkd6Q3,,,,, +https://oreilly.com,8YKBuv,,,,, +https://oup.com,ZneUUv,,,,, +https://ow.ly,JCAZYJ,,,,, +https://ox.ac.uk,chmqTm,,,,, +https://parallels.com,MQbUhe,,,,, +https://paypal.com,PfYRVH,,,,, +https://pbs.org,npHSZh,,,,, +https://phoca.cz,wAdJnk,,,,, +https://photobucket.com,JgCAdR,,,,, +https://php.net,Stc2u2,,,,, +https://phpbb.com,faVVSh,,,,, +https://pinterest.com,eFpX3J,,,,, +https://playstation.com,A2BNY8,,,,, +https://plesk.com,y5C7r2,,,,, +https://plos.org,uqmDvW,,,,, +https://prestashop.com,dUapWs,,,,, +https://prnewswire.com,NQ8yNz,,,,, +https://psu.edu,RB4tm7,,,,, +https://psychologytoday.com,mpvZ2B,,,,, +https://python.org,d4v9xP,,,,, +https://qq.com,weyAvx,,,,, +https://quantcast.com,3LwS7y,,,,, +https://rakuten.co.jp,sBfK3c,,,,, +https://rambler.ru,47Fr9v,,,,, +https://redcross.org,MU2g8X,,,,, +https://reddit.com,nMecXS,,,,, +https://reference.com,d2WBFd,,,,, +https://researchgate.net,YWDXcC,,,,, +https://reuters.com,UQddnp,,,,, +https://rs6.net,seekPM,,,,, +https://sagepub.com,NSJJny,,,,, +https://sakura.ne.jp,ufrtfD,,,,, +https://samsung.com,Krqaq3,,,,, +https://sciencedirect.com,Z43cwq,,,,, +https://sciencemag.org,FP5JEb,,,,, +https://scientificamerican.com,tmnQv3,,,,, +https://scribd.com,YCxcKX,,,,, +https://sec.gov,pAqrME,,,,, +https://secureserver.net,9cwX6F,,,,, +https://sedo.com,9wP2Zp,,,,, +https://sedoparking.com,YnQSB4,,,,, +https://senate.gov,wvyGhy,,,,, +https://shinystat.com,Vc7yAN,,,,, +https://shop-pro.jp,TgP2pT,,,,, +https://shopify.com,DZy7HN,,,,, +https://si.edu,gp56zv,,,,, +https://sina.com.cn,JdVEXg,,,,, +https://siteorigin.com,RugBuh,,,,, +https://skype.com,UAygVS,,,,, +https://slate.com,SZYUhQ,,,,, +https://slideshare.net,2wzL9T,,,,, +https://snapchat.com,nJpJGq,,,,, +https://sogou.com,QaLgNw,,,,, +https://sohu.com,MVF33U,,,,, +https://soundcloud.com,G28FFZ,,,,, +https://sourceforge.net,52k2zf,,,,, +https://spotify.com,n7FkPD,,,,, +https://springer.com,E77h2c,,,,, +https://squarespace.com,f224CV,,,,, +https://squareup.com,FgRKzW,,,,, +https://ssa.gov,Ts77ba,,,,, +https://stackoverflow.com,CknNa9,,,,, +https://stanford.edu,tkDWU5,,,,, +https://starwoodhotels.com,6nHck4,,,,, +https://statcounter.com,VbEb9p,,,,, +https://state.gov,MYKfHG,,,,, +https://steampowered.com,VhntJs,,,,, +https://storify.com,mkmkJJ,,,,, +https://studiopress.com,926bk2,,,,, +https://stumbleupon.com,g2KsDb,,,,, +https://sun.com,4RGN9a,,,,, +https://surveymonkey.com,xzqdF3,,,,, +https://symantec.com,EAKzdV,,,,, +https://t.co,66YaSw,,,,, +https://t.me,rTeJgm,,,,, +https://tandfonline.com,R7kX3b,,,,, +https://taobao.com,mRYb98,,,,, +https://teamviewer.com,xcqBLB,,,,, +https://techcrunch.com,Jtzmhh,,,,, +https://ted.com,x3caeP,,,,, +https://telegram.me,DwbCbq,,,,, +https://telegraph.co.uk,aKVBnt,,,,, +https://theatlantic.com,yMD6sw,,,,, +https://theguardian.com,DZeQLT,,,,, +https://thehill.com,MYt99N,,,,, +https://themeforest.net,8QFUk2,,,,, +https://themegrill.com,9f2b9r,,,,, +https://thenextweb.com,r2qB55,,,,, +https://theverge.com,EAb3vZ,,,,, +https://ticketmaster.com,KvGxLq,,,,, +https://time.com,rN34FS,,,,, +https://tmall.com,YHFG3t,,,,, +https://today.com,5TeUKe,,,,, +https://tripadvisor.co.uk,VvMzH5,,,,, +https://tripadvisor.com,Zk3Q25,,,,, +https://trustpilot.com,HMhEXJ,,,,, +https://tucowsdomains.com,rbYPR5,,,,, +https://tumblr.com,TaRf9k,,,,, +https://twitch.tv,UXCGYt,,,,, +https://twitter.com,RuCCwJ,,,,, +https://typeform.com,D54cHf,,,,, +https://typepad.com,F7HqtQ,,,,, +https://uchicago.edu,cuB2ww,,,,, +https://ucl.ac.uk,JAtXNr,,,,, +https://ucla.edu,gG6rSA,,,,, +https://umblr.com,4p9paT,,,,, +https://umich.edu,UBPfxn,,,,, +https://umn.edu,r6YqYx,,,,, +https://un.org,R39hnB,,,,, +https://unesco.org,kdBBsS,,,,, +https://unicef.org,Vwvfav,,,,, +https://unsplash.com,Fh4zbQ,,,,, +https://uol.com.br,sSNDLb,,,,, +https://upenn.edu,gdczmp,,,,, +https://usa.gov,nDkhkv,,,,, +https://usatoday.com,XQfHfR,,,,, +https://usc.edu,5m9RWH,,,,, +https://usda.gov,Gh6cv4,,,,, +https://usgs.gov,7DBnHW,,,,, +https://usnews.com,WH7nFv,,,,, +https://ustream.tv,qUztv2,,,,, +https://utexas.edu,5fLCFN,,,,, +https://va.gov,BbMAkx,,,,, +https://variety.com,dMwpeZ,,,,, +https://venturebeat.com,6C2p7f,,,,, +https://vice.com,sG6D8U,,,,, +https://vimeo.com,FuRbbQ,,,,, +https://visma.com,hE8QMY,,,,, +https://vk.com,GNmBrr,,,,, +https://vkontakte.ru,zaM7x8,,,,, +https://w3.org,FFaMrB,,,,, +https://w3schools.com,dnRCCf,,,,, +https://warnerbros.com,nu43eZ,,,,, +https://washington.edu,NPMEVq,,,,, +https://washingtonpost.com,nTuDP2,,,,, +https://web.de,kmNcgr,,,,, +https://webmd.com,r9Kx9g,,,,, +https://webs.com,95Lre4,,,,, +https://weebly.com,ccuBmQ,,,,, +https://weibo.com,QDVD6n,,,,, +https://whatsapp.com,rtUhP9,,,,, +https://whitehouse.gov,YQcRDE,,,,, +https://who.int,m6VAL9,,,,, +https://wikia.com,xQBScq,,,,, +https://wikihow.com,h6yr7K,,,,, +https://wikimedia.org,CVstJQ,,,,, +https://wikipedia.org,nn3LDm,,,,, +https://wiley.com,LD8WuS,,,,, +https://windowsphone.com,LhyHBM,,,,, +https://wired.com,UXkfGe,,,,, +https://wisc.edu,NGdchr,,,,, +https://wix.com,wLDq4G,,,,, +https://wixsite.com,fMpzev,,,,, +https://wordpress.com,ECpAm7,,,,, +https://wordpress.org,d68ksJ,,,,, +https://worldbank.org,PX79hx,,,,, +https://wp.com,J33BBN,,,,, +https://wp.me,brxd9r,,,,, +https://wsimg.com,zHJaAn,,,,, +https://wsj.com,fMX49B,,,,, +https://wufoo.com,2wsvNQ,,,,, +https://wunderground.com,SwQRET,,,,, +https://www.gov.uk,PgF38a,,,,, +https://www.nhs.uk,NYaMpe,,,,, +https://xing.com,7WFUKF,,,,, +https://xinhuanet.com,Lhk4Eh,,,,, +https://xiti.com,TYgcPD,,,,, +https://yahoo.co.jp,sKahCa,,,,, +https://yahoo.com,hQUFE7,,,,, +https://yale.edu,Wmppmv,,,,, +https://yandex.ru,QMW2aA,,,,, +https://yelp.com,pbL6hs,,,,, +https://youku.com,gMHZaU,,,,, +https://youronlinechoices.com,XdN7zZ,,,,, +https://youtu.be,JFeL8f,,,,, +https://youtube.com,emcsUH,,,,, +https://zdnet.com,a4S54H,,,,, +https://zendesk.com,bCvPgq,,,,, +https://zenfolio.com,namGpc,,,,, +hxHYLL,https://noaa.gov,2018-06-11T22:59:03Z,,,, +k2CqF3,https://mail.ru,2018-06-11T22:59:03Z,,,, +k99Re9,https://amzn.to,2018-06-11T22:59:03Z,,,, +kDrgJL,https://bbb.org,2018-06-11T22:59:03Z,,,, +kDzdzJ,https://buydomains.com,2018-06-11T22:59:03Z,,,, +kS6XCE,https://hostnet.nl,2018-06-11T22:59:03Z,,,, +kdBBsS,https://unesco.org,2018-06-11T22:59:03Z,,,, +kmNcgr,https://web.de,2018-06-11T22:59:03Z,,,, +krHE7C,https://chicagotribune.com,2018-06-11T22:59:03Z,,,, +m57dUp,https://a8.net,2018-06-11T22:59:03Z,,,, +m6VAL9,https://who.int,2018-06-11T22:59:03Z,,,, +m8nEGs,https://amazonaws.com,2018-06-11T22:59:03Z,,,, +mRYb98,https://taobao.com,2018-06-11T22:59:03Z,,,, +mb78SA,https://nist.gov,2018-06-11T22:59:03Z,,,, +mkmkJJ,https://storify.com,2018-06-11T22:59:03Z,,,, +mpvZ2B,https://psychologytoday.com,2018-06-11T22:59:03Z,,,, +muKXKA,https://harvard.edu,2018-06-11T22:59:03Z,,,, +n6uFCc,https://mynavi.jp,2018-06-11T22:59:03Z,,,, +n7FkPD,https://spotify.com,2018-06-11T22:59:03Z,,,, +nDkhkv,https://usa.gov,2018-06-11T22:59:03Z,,,, +nJZpMb,https://columbia.edu,2018-06-11T22:59:03Z,,,, +nJpJGq,https://snapchat.com,2018-06-11T22:59:03Z,,,, +nMecXS,https://reddit.com,2018-06-11T22:59:03Z,,,, +nTuDP2,https://washingtonpost.com,2018-06-11T22:59:03Z,,,, +nVgcGL,https://moodle.org,2018-06-11T22:59:03Z,,,, +namGpc,https://zenfolio.com,2018-06-11T22:59:03Z,,,, +nht265,https://europa.eu,2018-06-11T22:59:03Z,,,, +nn3LDm,https://wikipedia.org,2018-06-11T22:59:03Z,,,, +npHSZh,https://pbs.org,2018-06-11T22:59:03Z,,,, +nu43eZ,https://warnerbros.com,2018-06-11T22:59:03Z,,,, +p2mUXv,https://cnet.com,2018-06-11T22:59:03Z,,,, +pAqrME,https://sec.gov,2018-06-11T22:59:03Z,,,, +pEzkmx,https://eff.org,2018-06-11T22:59:03Z,,,, +pJyTHb,https://adobe.com,2018-06-11T22:59:03Z,,,, +pWWnAF,https://mashable.com,2018-06-11T22:59:03Z,,,, +pXmq2C,https://domainretailing.com,2018-06-11T22:59:03Z,,,, +pbL6hs,https://yelp.com,2018-06-11T22:59:03Z,,,, +ppRLR5,https://macromedia.com,2018-06-11T22:59:03Z,,,, +ptN9pf,https://163.com,2018-06-11T22:59:03Z,,,, +q7zbJ5,https://alibaba.com,2018-06-11T22:59:03Z,,,, +qJMJZy,https://one.com,2018-06-11T22:59:03Z,,,, +qUztv2,https://ustream.tv,2018-06-11T22:59:03Z,,,, +qVygK6,https://goo.ne.jp,2018-06-11T22:59:03Z,,,, +qh24JD,https://indiatimes.com,2018-06-11T22:59:03Z,,,, +qn6aVp,https://namejet.com,2018-06-11T22:59:03Z,,,, +qnD4XJ,https://goo.gl,2018-06-11T22:59:03Z,,,, +qqzgpC,https://java.com,2018-06-11T22:59:03Z,,,, +qr4CgU,https://amazon.de,2018-06-11T22:59:03Z,,,, +qs4EfZ,https://cmu.edu,2018-06-11T22:59:03Z,,,, +qv2R5m,https://bit.ly,2018-06-11T22:59:03Z,,,, +qyCWYV,https://house.gov,2018-06-11T22:59:03Z,,,, +r2qB55,https://thenextweb.com,2018-06-11T22:59:03Z,,,, +r6YqYx,https://umn.edu,2018-06-11T22:59:03Z,,,, +r8tcAG,https://domeneshop.no,2018-06-11T22:59:03Z,,,, +r9Kx9g,https://webmd.com,2018-06-11T22:59:03Z,,,, +rN34FS,https://time.com,2018-06-11T22:59:03Z,,,, +rTeJgm,https://t.me,2018-06-11T22:59:03Z,,,, +rUSgBk,https://fda.gov,2018-06-11T22:59:03Z,,,, +rWuJwU,https://ewebdevelopment.com,2018-06-11T22:59:03Z,,,, +rbYPR5,https://tucowsdomains.com,2018-06-11T22:59:03Z,,,, +rtUhP9,https://whatsapp.com,2018-06-11T22:59:03Z,,,, +s8h7y4,https://opensource.org,2018-06-11T22:59:03Z,,,, +sBDKBC,https://feedburner.com,2018-06-11T22:59:03Z,,,, +sBZ2am,https://eventbrite.com,2018-06-11T22:59:03Z,,,, +sBfK3c,https://rakuten.co.jp,2018-06-11T22:59:03Z,,,, +sEtApG,https://cam.ac.uk,2018-06-11T22:59:03Z,,,, +sG6D8U,https://vice.com,2018-06-11T22:59:03Z,,,, +sGNMLW,https://blogspot.com.es,2018-06-11T22:59:03Z,,,, +sHxY2E,https://go.com,2018-06-11T22:59:03Z,,,, +sKahCa,https://yahoo.co.jp,2018-06-11T22:59:03Z,,,, +sLYcUG,https://bandcamp.com,2018-06-11T22:59:03Z,,,, +sPGEkh,https://hhs.gov,2018-06-11T22:59:03Z,,,, +sSNDLb,https://uol.com.br,2018-06-11T22:59:03Z,,,, +sSzEZm,https://hibu.com,2018-06-11T22:59:03Z,,,, +sVCfW8,https://ietf.org,2018-06-11T22:59:03Z,,,, +seekPM,https://rs6.net,2018-06-11T22:59:03Z,,,, +srddMa,https://delicious.com,2018-06-11T22:59:03Z,,,, +t6FKQa,https://msdn.com,2018-06-11T22:59:03Z,,,, +t8F4qH,https://irs.gov,2018-06-11T22:59:03Z,,,, +tHPaHp,https://microsoft.com,2018-06-11T22:59:03Z,,,, +tJBpkh,https://justgiving.com,2018-06-11T22:59:03Z,,,, +tM2RBF,https://google.co.in,2018-06-11T22:59:03Z,,,, +tYcFwg,https://fb.me,2018-06-11T22:59:03Z,,,, +te7adS,https://google.be,2018-06-11T22:59:03Z,,,, +tkCTth,https://bluehost.com,2018-06-11T22:59:03Z,,,, +tkDWU5,https://stanford.edu,2018-06-11T22:59:03Z,,,, +tmnQv3,https://scientificamerican.com,2018-06-11T22:59:03Z,,,, +u2T9RE,https://guardian.co.uk,2018-06-11T22:59:03Z,,,, +u8z9xW,https://bloomberg.com,2018-06-11T22:59:03Z,,,, +uBpHuF,https://netscape.com,2018-06-11T22:59:03Z,,,, +uR3PMX,https://ok.ru,2018-06-11T22:59:03Z,,,, +uWB3RB,https://businesswire.com,2018-06-11T22:59:03Z,,,, +uYWA7x,https://deviantart.com,2018-06-11T22:59:03Z,,,, +ufdmPd,https://amazon.co.jp,2018-06-11T22:59:03Z,,,, +ufrtfD,https://sakura.ne.jp,2018-06-11T22:59:03Z,,,, +ugKHdK,https://nasa.gov,2018-06-11T22:59:03Z,,,, +umkWBr,https://loopia.se,2018-06-11T22:59:03Z,,,, +un9Pkg,https://bigcommerce.com,2018-06-11T22:59:03Z,,,, +uqmDvW,https://plos.org,2018-06-11T22:59:03Z,,,, +vEmdB8,https://biomedcentral.com,2018-06-11T22:59:03Z,,,, +vEu4Th,https://google.es,2018-06-11T22:59:03Z,,,, +vkFbaR,https://cnn.com,2018-06-11T22:59:03Z,,,, +vkaedE,https://blogspot.co.uk,2018-06-11T22:59:03Z,,,, +vsbRsK,https://hubspot.com,2018-06-11T22:59:03Z,,,, +vtWG9W,https://google.ch,2018-06-11T22:59:03Z,,,, +vzkZh7,https://list-manage1.com,2018-06-11T22:59:03Z,,,, +w4dDcB,https://behance.net,2018-06-11T22:59:03Z,,,, +wAdJnk,https://phoca.cz,2018-06-11T22:59:03Z,,,, +wF2uVC,https://msn.com,2018-06-11T22:59:03Z,,,, +wHCbXq,https://nationalgeographic.com,2018-06-11T22:59:03Z,,,, +wKxV53,https://bigcartel.com,2018-06-11T22:59:03Z,,,, +wLDq4G,https://wix.com,2018-06-11T22:59:03Z,,,, +wWkkv8,https://apple.com,2018-06-11T22:59:03Z,,,, +weyAvx,https://qq.com,2018-06-11T22:59:03Z,,,, +wvyGhy,https://senate.gov,2018-06-11T22:59:03Z,,,, +wyUCAe,https://51.la,2018-06-11T22:59:03Z,,,, +x3caeP,https://ted.com,2018-06-11T22:59:03Z,,,, +x9EbEB,https://goodreads.com,2018-06-11T22:59:03Z,,,, +xJ69AD,https://mhlw.go.jp,2018-06-11T22:59:03Z,,,, +xNzmwV,https://fb.com,2018-06-11T22:59:03Z,,,, +xQBScq,https://wikia.com,2018-06-11T22:59:03Z,,,, +xUBfwT,https://hatena.ne.jp,2018-06-11T22:59:03Z,,,, +xcL2KB,https://japanpost.jp,2018-06-11T22:59:03Z,,,, +xcqBLB,https://teamviewer.com,2018-06-11T22:59:03Z,,,, +xsdVgM,https://fcc.gov,2018-06-11T22:59:03Z,,,, +xzqdF3,https://surveymonkey.com,2018-06-11T22:59:03Z,,,, +y5C7r2,https://plesk.com,2018-06-11T22:59:03Z,,,, +yBNq4m,https://addthis.com,2018-06-11T22:59:03Z,,,, +yDVbFL,https://mlit.go.jp,2018-06-11T22:59:03Z,,,, +yDgRDy,https://enable-javascript.com,2018-06-11T22:59:03Z,,,, +yFxM7M,https://github.com,2018-06-11T22:59:03Z,,,, +yMD6sw,https://theatlantic.com,2018-06-11T22:59:03Z,,,, +yMFySb,https://google.com.au,2018-06-11T22:59:03Z,,,, +yMShaY,https://google.co.jp,2018-06-11T22:59:03Z,,,, +ySvWBH,https://allaboutcookies.org,2018-06-11T22:59:03Z,,,, +yY2DGh,https://change.org,2018-06-11T22:59:03Z,,,, +yqqECD,https://nytimes.com,2018-06-11T22:59:03Z,,,, +ysGShJ,https://economist.com,2018-06-11T22:59:03Z,,,, +z9ZHQQ,https://dailymotion.com,2018-06-11T22:59:03Z,,,, +zHJaAn,https://wsimg.com,2018-06-11T22:59:03Z,,,, +zYu9LW,https://ibm.com,2018-06-11T22:59:03Z,,,, +zaJPJ2,https://iso.org,2018-06-11T22:59:03Z,,,, +zaM7x8,https://vkontakte.ru,2018-06-11T22:59:03Z,,,, +zagnaG,https://eepurl.com,2018-06-11T22:59:03Z,,,, +zmnEUS,https://businessinsider.com,2018-06-11T22:59:03Z,,,, +zyPwNq,https://aliyun.com,2018-06-11T22:59:03Z,,,, diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml index d106ab9..bf67021 100644 --- a/examples/docker-compose.yml +++ b/examples/docker-compose.yml @@ -1,13 +1,18 @@ +# +# sample docker-compose configuration file +# requires: +# - a valid settings.yaml file +# version: '3.1' services: - clerk: + distill: image: registry.gitlab.com/welance/oss/distill:latest container_name: distill restart: unless-stopped - networks: - - local ports: - 1804:1804 volumes: - - ./distil.db:/data/distill.db - - ./settings.yml:/data/configs/settings.docker.yaml + - ./distil.db:/data/db + - ./settings.yaml:/data/configs/settings.docker.yaml + + -- GitLab From 4d029140020b93b63ebd2e95c755744c8a325b26 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Fri, 22 Jun 2018 17:41:55 +0200 Subject: [PATCH 31/35] support kutt.it api interface --- internal/distill/endpoints.go | 56 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/internal/distill/endpoints.go b/internal/distill/endpoints.go index 820f92a..6258b79 100644 --- a/internal/distill/endpoints.go +++ b/internal/distill/endpoints.go @@ -67,32 +67,9 @@ func RegisterEndpoints() (router *chi.Mux) { http.Error(w, "URL not found", 404) }) // handle url setup - r.Post("/short", func(w http.ResponseWriter, r *http.Request) { - urlReq := &URLReq{} - if err := render.Bind(r, urlReq); err != nil { - render.Render(w, r, ErrInvalidRequest(err, err.Error())) - return - } - // retrieve the forceAlphabet and forceLength - forceAlphabet, forceLenght := false, false - fA := chi.URLParam(r, "forceAlphabet") - fL := chi.URLParam(r, "forceLenght") - if fA == "1" { - forceAlphabet = true - } - if fL == "1" { - forceLenght = true - } - // upsert the data - id, err := UpsertURL(urlReq, forceAlphabet, forceLenght, time.Now()) - mlog.Trace("creted %v", id) - // TODO: check the actual error - if err != nil { - render.Render(w, r, ErrInvalidRequest(err, err.Error())) - return - } - render.JSON(w, r, ShortID{ID: id}) - }) + r.Post("/short", handleShort) + // implement kutt.it endpoint + r.Post("/url/submit", handleShort) // delete an id r.Delete("/short/{ID}", func(w http.ResponseWriter, r *http.Request) { @@ -117,6 +94,33 @@ func RegisterEndpoints() (router *chi.Mux) { // |____||____||____| |____||_____|\____||______.'|________||________||____| |___|\______.' // +func handleShort(w http.ResponseWriter, r *http.Request) { + urlReq := &URLReq{} + if err := render.Bind(r, urlReq); err != nil { + render.Render(w, r, ErrInvalidRequest(err, err.Error())) + return + } + // retrieve the forceAlphabet and forceLength + forceAlphabet, forceLenght := false, false + fA := chi.URLParam(r, "forceAlphabet") + fL := chi.URLParam(r, "forceLenght") + if fA == "1" { + forceAlphabet = true + } + if fL == "1" { + forceLenght = true + } + // upsert the data + id, err := UpsertURL(urlReq, forceAlphabet, forceLenght, time.Now()) + mlog.Trace("creted %v", id) + // TODO: check the actual error + if err != nil { + render.Render(w, r, ErrInvalidRequest(err, err.Error())) + return + } + render.JSON(w, r, ShortID{ID: id}) +} + func handleGetURL(w http.ResponseWriter, r *http.Request) { shortID := chi.URLParam(r, "ID") targetURL, err := GetURLRedirect(shortID) -- GitLab From b2f675b6854cff34169dbf62a88c1e83146bacc4 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Sat, 23 Jun 2018 23:50:22 +0200 Subject: [PATCH 32/35] Refactored how statistics are collected (work in progress) --- internal/config.go | 2 - internal/distill/distill_test.go | 22 ++--- internal/distill/model.go | 2 - internal/distill/stats.go | 133 +++++++++---------------------- internal/distill/stats_test.go | 3 +- internal/distill/urlstore.go | 42 ++++++++++ pkg/common/helpers_test.go | 4 +- 7 files changed, 96 insertions(+), 112 deletions(-) diff --git a/internal/config.go b/internal/config.go index f60b7af..dc6c7f1 100644 --- a/internal/config.go +++ b/internal/config.go @@ -32,7 +32,6 @@ type ShortIDConfig struct { // TuningConfig fine tuning configuration type TuningConfig struct { StatsEventsWorkerNum int `yaml:"statsEventsWorkerNum"` - StatsEventsQueueSize int `yaml:"statsEventsQueueSize"` StatsCaheSize int `yaml:"statsCacheSize"` DbPurgeWritesCount int `yaml:"dbPurgeWritesCount"` DbGCDeletesCount int `yaml:"dbGCDeletesCount"` @@ -67,7 +66,6 @@ func (c *ConfigSchema) Defaults() { common.DefaultIfEmptyInt(&c.ShortID.Length, 6) // For tuning - common.DefaultIfEmptyInt(&c.Tuning.StatsEventsQueueSize, 2048) common.DefaultIfEmptyInt(&c.Tuning.StatsEventsWorkerNum, 1) common.DefaultIfEmptyInt(&c.Tuning.StatsCaheSize, 1024) common.DefaultIfEmptyInt(&c.Tuning.DbPurgeWritesCount, 2000) diff --git a/internal/distill/distill_test.go b/internal/distill/distill_test.go index 0441ba0..f6c9963 100644 --- a/internal/distill/distill_test.go +++ b/internal/distill/distill_test.go @@ -487,7 +487,7 @@ func TestExpireRequestsUrl(t *testing.T) { }) } // get the stats - time.Sleep(time.Duration(10) * time.Millisecond) + //TODO: time.Sleep(time.Duration(10) * time.Millisecond) s := GetStats() t.Log(s) if s.Urls != 2 { @@ -503,16 +503,16 @@ func TestExpireTTLUrl(t *testing.T) { wantErr bool }{ { - name: "noexpire", + name: "noexpire w1", wantErr: false, wait: 1, param: URLReq{ - URL: "https://ilij.li/?param=noexpire", + URL: "https://ilij.li/?param=noexpireA", TTL: 0, }, }, { - name: "noexpire", + name: "noexpire w2", wantErr: false, wait: 2, param: URLReq{ @@ -522,7 +522,7 @@ func TestExpireTTLUrl(t *testing.T) { }, }, { - name: "expire1", + name: "expire w3", wantErr: true, wait: 3, param: URLReq{ @@ -531,9 +531,9 @@ func TestExpireTTLUrl(t *testing.T) { }, }, { - name: "expire10", + name: "expire w4", wantErr: true, - wait: 3, + wait: 4, param: URLReq{ URL: "https://ilij.li/?param=expire10", MaxRequests: 2, @@ -548,11 +548,14 @@ func TestExpireTTLUrl(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { id, err := UpsertURL(&tt.param, true, true, time.Now()) - mlog.Info("-- upsert %s --", id) + //mlog.Info("-- upsert %s --", id) // consume all the requests time.Sleep(time.Duration(tt.wait) * time.Second) + now := time.Now() _, err = GetURLRedirect(id) - mlog.Info("-- << end %s --", id) + u, _ := GetURLInfo(id) + fmt.Println(tt.name, tt.wantErr, "\nbat", u.BountAt, "\nexp", u.ExpireOn, "\nnow", now.UTC(), "\ndif", now.Sub(u.BountAt)) + //mlog.Info("-- << end %s --", id) // this should be a not found now for the expired hasErr := (err != nil) if tt.wantErr != hasErr { @@ -562,7 +565,6 @@ func TestExpireTTLUrl(t *testing.T) { }) } // get the stats - time.Sleep(time.Duration(10) * time.Millisecond) s := GetStats() t.Log(s) if s.Urls != 2 { diff --git a/internal/distill/model.go b/internal/distill/model.go index 0dabaa2..6713acb 100644 --- a/internal/distill/model.go +++ b/internal/distill/model.go @@ -4,7 +4,6 @@ import ( "encoding/binary" "fmt" "strconv" - "sync" "time" ) @@ -61,7 +60,6 @@ type URLReq struct { // Statistics contains the global statistics type Statistics struct { - mutex sync.Mutex Urls int64 `json:"urls"` Gets int64 `json:"gets"` Upserts int64 `json:"upserts"` diff --git a/internal/distill/stats.go b/internal/distill/stats.go index 45ea832..3401ede 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -20,25 +20,17 @@ var ( // sytem keys sysKeyPurgeCount []byte sysKeyGCCount []byte - // stats keys - statsKeyGlobalURLCount []byte - statsKeyGlobalGetCount []byte - statsKeyGlobalDelCount []byte - statsKeyGlobalUpdCount []byte ) // NewStatistics starts the statistics collector worker pool func NewStatistics() (err error) { // initialize system key - sysKeyPurgeCount = keySys("ilij_sys_purge_count") - sysKeyGCCount = keySys("ilij_sys_gc_count") - // initialize stats keys - statsKeyGlobalURLCount = keyGlobalStat("ilij_global_url_count") - statsKeyGlobalGetCount = keyGlobalStat("ilij_global_get_count") - statsKeyGlobalDelCount = keyGlobalStat("ilij_global_del_count") - statsKeyGlobalUpdCount = keyGlobalStat("ilij_global_upd_count") + sysKeyPurgeCount = keySys("distill_sys_purge_count") + sysKeyGCCount = keySys("distill_sys_gc_count") + // read the current statistics - globalStatistics, err = loadGlobalStatistics() + globalStatistics = &Statistics{} + err = loadGlobalStatistics(globalStatistics) if err != nil { return } @@ -47,8 +39,8 @@ func NewStatistics() (err error) { ARC(). Build() // Initialize channel of events - mlog.Trace("intialize queue size %d", internal.Config.Tuning.StatsEventsQueueSize) - opEventsQueue = make(chan *URLOp, internal.Config.Tuning.StatsEventsQueueSize) + mlog.Trace("intialize events queue") + opEventsQueue = make(chan *URLOp) // start the routines for i := 0; i < internal.Config.Tuning.StatsEventsWorkerNum; i++ { wg.Add(1) @@ -71,22 +63,17 @@ func GetStats() (s *Statistics) { return globalStatistics } -func loadGlobalStatistics() (s *Statistics, err error) { - s = &Statistics{} - err = db.View(func(txn *badger.Txn) (err error) { - s.Urls = dbGetInt64(txn, statsKeyGlobalURLCount) - s.Gets = dbGetInt64(txn, statsKeyGlobalGetCount) - s.Deletes = dbGetInt64(txn, statsKeyGlobalDelCount) - s.Upserts = dbGetInt64(txn, statsKeyGlobalUpdCount) - return - }) - globalStatistics = s +func loadGlobalStatistics(s *Statistics) (err error) { + err = LoadStats(s) + if err != nil { + mlog.Error(err) + } mlog.Info("Status: %v", s) return } func resetGlobalStatistics() (err error) { - s := &Statistics{} + globalStatistics := &Statistics{} // iterate over the urls i := NewURLIterator() for i.HasNext() { @@ -94,23 +81,14 @@ func resetGlobalStatistics() (err error) { if err != nil { mlog.Warning("Warning looping through the URLs") } - s.Urls++ - s.Upserts++ - s.Gets += u.Counter + globalStatistics.Urls++ + globalStatistics.Upserts++ + globalStatistics.Gets += u.Counter } // close the iterator i.Close() // run the update - err = db.Update(func(txn *badger.Txn) (err error) { - // find all the urls - dbSetInt64(txn, statsKeyGlobalURLCount, s.Urls) - dbSetInt64(txn, statsKeyGlobalGetCount, s.Gets) - dbSetInt64(txn, statsKeyGlobalDelCount, 0) - dbSetInt64(txn, statsKeyGlobalUpdCount, s.Upserts) - // update global statistics - globalStatistics = s - return - }) + err = SaveStats(globalStatistics) if err != nil { mlog.Warning("Error while rest stats %v", err) } @@ -124,8 +102,11 @@ func pushEvent(urlop *URLOp) { key := fmt.Sprint(urlop.opcode, ":", urlop.ID) // if it's in cache do not queue the job if _, err := sc.GetIFPresent(key); err == nil { + // if not set a nil value for some seconds so it + sc.SetWithExpire(key, nil, time.Duration(60)*time.Second) return } + // if not set a nil value for some seconds so it sc.SetWithExpire(key, nil, time.Duration(60)*time.Second) case opcodeInsert: for _, key := range []int{opcodeDelete, opcodeExpired} { @@ -146,51 +127,17 @@ func processEvents(workerID int) { mlog.Trace(">>> Event pid: %v, opcode:%v %v", workerID, opcodeToString(uo.opcode), uo.ID) switch uo.opcode { case opcodeGet: - db.Update(func(txn *badger.Txn) (err error) { - // update global gets count - globalStatistics.recGet() - dbSetInt64(txn, statsKeyGlobalGetCount, globalStatistics.Gets) - return - }) + globalStatistics.record(1, 0, 0, 0) case opcodeInsert: - db.Update(func(txn *badger.Txn) (err error) { - globalStatistics.recUpsert() - // update urls count - dbSetInt64(txn, statsKeyGlobalURLCount, globalStatistics.Urls) - // update upserts count - dbSetInt64(txn, statsKeyGlobalUpdCount, globalStatistics.Upserts) - return - }) + // TODO: check if existed already + globalStatistics.record(0, 1, 0, 1) case opcodeDelete: - db.Update(func(txn *badger.Txn) (err error) { - globalStatistics.recDelete() - // update urls count - dbSetInt64(txn, statsKeyGlobalURLCount, globalStatistics.Urls) - // update deletes count - dbSetInt64(txn, statsKeyGlobalDelCount, globalStatistics.Deletes) - return - }) + globalStatistics.record(0, 0, 1, -1) case opcodeExpired: - db.Update(func(txn *badger.Txn) (err error) { - // first verify that the url has not been already deleted - err = Delete(uo.ID) - if err != nil { - return - } - globalStatistics.recDelete() - // update urls count - dbSetInt64(txn, statsKeyGlobalURLCount, globalStatistics.Urls) - // update deletes count - dbSetInt64(txn, statsKeyGlobalDelCount, globalStatistics.Deletes) - return - }) + globalStatistics.record(0, 0, 1, -1) } mlog.Trace("<<< Event pid: %v, opcode:%v %v", workerID, opcodeToString(uo.opcode), uo.ID) - // run the maintenance - // TODO: this has to be fixed since it can create issues with the database (hint: use a channel) - // it can happen that the database get closed while the maintenance is about to run - // creating a panic - // go runDbMaintenance() + SaveStats(globalStatistics) } // complete task mlog.Trace("Stop event processor id: %v", workerID) @@ -198,24 +145,18 @@ func processEvents(workerID int) { } -func (s *Statistics) recUpsert() { - s.mutex.Lock() - s.Upserts++ - s.Urls++ - s.mutex.Unlock() -} - -func (s *Statistics) recDelete() { - s.mutex.Lock() - s.Deletes++ - s.Urls-- - s.mutex.Unlock() -} +// runDbMaintenance +var ( + statsMutex sync.Mutex +) -func (s *Statistics) recGet() { - s.mutex.Lock() - s.Gets++ - s.mutex.Unlock() +func (s *Statistics) record(get, upsert, delete, urls int64) { + statsMutex.Lock() + s.Gets += get + s.Upserts += upsert + s.Deletes += delete + s.Urls += urls + statsMutex.Unlock() } // runDbMaintenance diff --git a/internal/distill/stats_test.go b/internal/distill/stats_test.go index 5b97191..c14339c 100644 --- a/internal/distill/stats_test.go +++ b/internal/distill/stats_test.go @@ -72,7 +72,8 @@ func Test_loadGlobalStatistics(t *testing.T) { } CloseSession() NewSession() - gotS, err := loadGlobalStatistics() + gotS := &Statistics{} + err := loadGlobalStatistics(gotS) if (err != nil) != tt.wantErr { t.Errorf("loadGlobalStatistics() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/internal/distill/urlstore.go b/internal/distill/urlstore.go index 1d400d5..8158b09 100644 --- a/internal/distill/urlstore.go +++ b/internal/distill/urlstore.go @@ -18,6 +18,14 @@ const ( backupExtCsv = ".csv" ) +var ( + // initialize stats keys + statsKeyGlobalURLCount = keyGlobalStat("distill_global_url_count") + statsKeyGlobalGetCount = keyGlobalStat("distill_global_get_count") + statsKeyGlobalDelCount = keyGlobalStat("distill_global_del_count") + statsKeyGlobalUpdCount = keyGlobalStat("distill_global_upd_count") +) + var ( db *badger.DB uc gcache.Cache @@ -55,11 +63,45 @@ func CloseSession() { db.Close() } +// whenRemoved gets called by the memory cache +// it will check the value, if the value is nil +// means that the key has been deleted +// so it will delete it also from the persistent store func whenRemoved(key, value interface{}) { + if value == nil { + Delete(key.(string)) + return + } ui := value.(*URLInfo) Upsert(ui) } +// SaveStats write the URL's statistics +func SaveStats(s *Statistics) (err error) { + err = db.Update(func(txn *badger.Txn) (err error) { + // find all the urls + dbSetInt64(txn, statsKeyGlobalURLCount, s.Urls) + dbSetInt64(txn, statsKeyGlobalGetCount, s.Gets) + dbSetInt64(txn, statsKeyGlobalDelCount, s.Deletes) + dbSetInt64(txn, statsKeyGlobalUpdCount, s.Upserts) + // update global statistics + return + }) + return +} + +// LoadStats write the URL's statistics +func LoadStats(s *Statistics) (err error) { + err = db.View(func(txn *badger.Txn) (err error) { + s.Urls = dbGetInt64(txn, statsKeyGlobalURLCount) + s.Gets = dbGetInt64(txn, statsKeyGlobalGetCount) + s.Deletes = dbGetInt64(txn, statsKeyGlobalDelCount) + s.Upserts = dbGetInt64(txn, statsKeyGlobalUpdCount) + return + }) + return +} + // Insert an url into the the urlstore func Insert(u *URLInfo) (err error) { err = db.Update(func(txn *badger.Txn) (err error) { diff --git a/pkg/common/helpers_test.go b/pkg/common/helpers_test.go index 97f1604..6350a60 100644 --- a/pkg/common/helpers_test.go +++ b/pkg/common/helpers_test.go @@ -1,6 +1,8 @@ package common -import "testing" +import ( + "testing" +) func TestIsEmptyStr(t *testing.T) { type args struct { -- GitLab From 5824569e09783ac39236c23ef5db366248d2f772 Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 25 Jun 2018 23:54:03 +0200 Subject: [PATCH 33/35] Simplify the statistics management --- internal/distill/endpoints.go | 2 +- internal/distill/stats.go | 120 +++++++++------------------------ internal/distill/stats_test.go | 7 +- internal/distill/urlstore.go | 74 ++++++++++++++++---- 4 files changed, 95 insertions(+), 108 deletions(-) diff --git a/internal/distill/endpoints.go b/internal/distill/endpoints.go index 6258b79..1bd83dc 100644 --- a/internal/distill/endpoints.go +++ b/internal/distill/endpoints.go @@ -49,7 +49,7 @@ func RegisterEndpoints() (router *chi.Mux) { render.JSON(w, r, *GetStats()) }) r.Delete("/stats", func(w http.ResponseWriter, r *http.Request) { - err := resetGlobalStatistics() + err := ResetStats() if err != nil { render.Render(w, r, ErrInternalError(err, err.Error())) return diff --git a/internal/distill/stats.go b/internal/distill/stats.go index 3401ede..8facb84 100644 --- a/internal/distill/stats.go +++ b/internal/distill/stats.go @@ -1,9 +1,7 @@ package distill import ( - "fmt" "sync" - "time" "github.com/bluele/gcache" "github.com/jbrodriguez/mlog" @@ -22,98 +20,41 @@ var ( sysKeyGCCount []byte ) -// NewStatistics starts the statistics collector worker pool -func NewStatistics() (err error) { - // initialize system key - sysKeyPurgeCount = keySys("distill_sys_purge_count") - sysKeyGCCount = keySys("distill_sys_gc_count") - - // read the current statistics - globalStatistics = &Statistics{} - err = loadGlobalStatistics(globalStatistics) - if err != nil { - return - } - // initialzie internal cache to provide idempotency for events - sc = gcache.New(internal.Config.Tuning.StatsCaheSize). - ARC(). - Build() - // Initialize channel of events - mlog.Trace("intialize events queue") - opEventsQueue = make(chan *URLOp) - // start the routines - for i := 0; i < internal.Config.Tuning.StatsEventsWorkerNum; i++ { - wg.Add(1) - go processEvents(i) - } - - return -} - -// StopStatistics stops the statistics -func StopStatistics() { - // stop the running - mlog.Info("Stop statistics, lost %d events", len(opEventsQueue)) - close(opEventsQueue) - wg.Wait() -} - -// GetStats retrieve the global statistics -func GetStats() (s *Statistics) { - return globalStatistics -} - -func loadGlobalStatistics(s *Statistics) (err error) { - err = LoadStats(s) - if err != nil { - mlog.Error(err) - } - mlog.Info("Status: %v", s) - return -} - -func resetGlobalStatistics() (err error) { - globalStatistics := &Statistics{} - // iterate over the urls - i := NewURLIterator() - for i.HasNext() { - u, err := i.NextURL() - if err != nil { - mlog.Warning("Warning looping through the URLs") - } - globalStatistics.Urls++ - globalStatistics.Upserts++ - globalStatistics.Gets += u.Counter - } - // close the iterator - i.Close() - // run the update - err = SaveStats(globalStatistics) - if err != nil { - mlog.Warning("Error while rest stats %v", err) - } - return -} - // pushEvent in the url operaiton queue func pushEvent(urlop *URLOp) { + s := Statistics{} switch urlop.opcode { - case opcodeDelete, opcodeExpired: - key := fmt.Sprint(urlop.opcode, ":", urlop.ID) - // if it's in cache do not queue the job - if _, err := sc.GetIFPresent(key); err == nil { - // if not set a nil value for some seconds so it - sc.SetWithExpire(key, nil, time.Duration(60)*time.Second) - return - } - // if not set a nil value for some seconds so it - sc.SetWithExpire(key, nil, time.Duration(60)*time.Second) + case opcodeDelete: + s.Deletes++ + s.Urls-- case opcodeInsert: - for _, key := range []int{opcodeDelete, opcodeExpired} { - sc.Remove(key) - } + s.Upserts++ + s.Urls++ + case opcodeGet: + s.Gets++ + case opcodeExpired: + s.Deletes++ + s.Urls-- + Delete(urlop.ID) } - opEventsQueue <- urlop + UpdateStats(s) + // switch urlop.opcode { + // case opcodeDelete, opcodeExpired: + // key := fmt.Sprint(urlop.opcode, ":", urlop.ID) + // // if it's in cache do not queue the job + // if _, err := sc.GetIFPresent(key); err == nil { + // // if not set a nil value for some seconds so it + // sc.SetWithExpire(key, nil, time.Duration(60)*time.Second) + // return + // } + // // if not set a nil value for some seconds so it + // sc.SetWithExpire(key, nil, time.Duration(60)*time.Second) + // case opcodeInsert: + // for _, key := range []int{opcodeDelete, opcodeExpired} { + // sc.Remove(key) + // } + // } + //opEventsQueue <- urlop } // Process is an implementation of wp.Job.Process() @@ -137,7 +78,6 @@ func processEvents(workerID int) { globalStatistics.record(0, 0, 1, -1) } mlog.Trace("<<< Event pid: %v, opcode:%v %v", workerID, opcodeToString(uo.opcode), uo.ID) - SaveStats(globalStatistics) } // complete task mlog.Trace("Stop event processor id: %v", workerID) diff --git a/internal/distill/stats_test.go b/internal/distill/stats_test.go index c14339c..e8d6f9d 100644 --- a/internal/distill/stats_test.go +++ b/internal/distill/stats_test.go @@ -72,8 +72,9 @@ func Test_loadGlobalStatistics(t *testing.T) { } CloseSession() NewSession() - gotS := &Statistics{} - err := loadGlobalStatistics(gotS) + + err := LoadStats() + gotS := GetStats() if (err != nil) != tt.wantErr { t.Errorf("loadGlobalStatistics() error = %v, wantErr %v", err, tt.wantErr) return @@ -84,7 +85,7 @@ func Test_loadGlobalStatistics(t *testing.T) { // alsot test reset gotS.Deletes = 0 - err = resetGlobalStatistics() + err = ResetStats() if err != nil { t.Errorf("resetGlobalStatistics() error = %v, wantErr %v", err, false) return diff --git a/internal/distill/urlstore.go b/internal/distill/urlstore.go index 8158b09..e37f6e2 100644 --- a/internal/distill/urlstore.go +++ b/internal/distill/urlstore.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "sync" "github.com/bluele/gcache" "github.com/dgraph-io/badger" @@ -27,8 +28,10 @@ var ( ) var ( - db *badger.DB - uc gcache.Cache + db *badger.DB + uc gcache.Cache + st *Statistics + stM sync.Mutex ) // NewSession opens the underling storage @@ -50,7 +53,7 @@ func NewSession() { ARC(). Build() // inintialize statistics - err = NewStatistics() + err = LoadStats() if err != nil { mlog.Fatal(err) } @@ -58,7 +61,7 @@ func NewSession() { // CloseSession closes the underling storage func CloseSession() { - StopStatistics() + SaveStats() uc.Purge() db.Close() } @@ -77,13 +80,13 @@ func whenRemoved(key, value interface{}) { } // SaveStats write the URL's statistics -func SaveStats(s *Statistics) (err error) { +func SaveStats() (err error) { err = db.Update(func(txn *badger.Txn) (err error) { // find all the urls - dbSetInt64(txn, statsKeyGlobalURLCount, s.Urls) - dbSetInt64(txn, statsKeyGlobalGetCount, s.Gets) - dbSetInt64(txn, statsKeyGlobalDelCount, s.Deletes) - dbSetInt64(txn, statsKeyGlobalUpdCount, s.Upserts) + dbSetInt64(txn, statsKeyGlobalURLCount, st.Urls) + dbSetInt64(txn, statsKeyGlobalGetCount, st.Gets) + dbSetInt64(txn, statsKeyGlobalDelCount, st.Deletes) + dbSetInt64(txn, statsKeyGlobalUpdCount, st.Upserts) // update global statistics return }) @@ -91,17 +94,60 @@ func SaveStats(s *Statistics) (err error) { } // LoadStats write the URL's statistics -func LoadStats(s *Statistics) (err error) { +func LoadStats() (err error) { + // initialize object + st = &Statistics{} err = db.View(func(txn *badger.Txn) (err error) { - s.Urls = dbGetInt64(txn, statsKeyGlobalURLCount) - s.Gets = dbGetInt64(txn, statsKeyGlobalGetCount) - s.Deletes = dbGetInt64(txn, statsKeyGlobalDelCount) - s.Upserts = dbGetInt64(txn, statsKeyGlobalUpdCount) + st.Urls = dbGetInt64(txn, statsKeyGlobalURLCount) + st.Gets = dbGetInt64(txn, statsKeyGlobalGetCount) + st.Deletes = dbGetInt64(txn, statsKeyGlobalDelCount) + st.Upserts = dbGetInt64(txn, statsKeyGlobalUpdCount) return }) return } +// UpdateStats uppdate urls statistics +func UpdateStats(s Statistics) { + stM.Lock() + defer stM.Unlock() + st.Urls += s.Urls + st.Gets += s.Gets + st.Deletes += s.Deletes + st.Upserts += s.Upserts +} + +// ResetStats reset global statistcs +func ResetStats() (err error) { + stM.Lock() + defer stM.Unlock() + st = &Statistics{} + // iterate over the urls + i := NewURLIterator() + for i.HasNext() { + u, err := i.NextURL() + if err != nil { + mlog.Warning("Warning looping through the URLs") + } + st.Urls++ + st.Upserts++ + st.Gets += u.Counter + } + // close the iterator + i.Close() + // run the update + err = SaveStats() + if err != nil { + mlog.Warning("Error while rest stats %v", err) + } + return +} + +// GetStats get the statistics +func GetStats() (s *Statistics) { + return st +} + // Insert an url into the the urlstore func Insert(u *URLInfo) (err error) { err = db.Update(func(txn *badger.Txn) (err error) { -- GitLab From 8cd165bd7f2598ada68d5c7286b71bed1b670ace Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Mon, 25 Jun 2018 23:54:27 +0200 Subject: [PATCH 34/35] Add race test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 704e4a0..6692a11 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ build-dist: $(GOFILES) test: test-all test-all: - @go test -v $(GOPACKAGES) -coverprofile .testCoverage.txt + @go test -v -race $(GOPACKAGES) -coverprofile .testCoverage.txt bench: bench-all -- GitLab From 1d481fc9894d0c5010e6f6884832ddb17091d12b Mon Sep 17 00:00:00 2001 From: Andrea Giacobino Date: Tue, 26 Jun 2018 00:01:25 +0200 Subject: [PATCH 35/35] Remove race test since is making issues --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6692a11..704e4a0 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ build-dist: $(GOFILES) test: test-all test-all: - @go test -v -race $(GOPACKAGES) -coverprofile .testCoverage.txt + @go test -v $(GOPACKAGES) -coverprofile .testCoverage.txt bench: bench-all -- GitLab