From 47bcf5f2764e4ece11e9864e70f259a2059c57c7 Mon Sep 17 00:00:00 2001 From: Ben Fiedler Date: Wed, 7 Aug 2024 16:40:33 +0200 Subject: [PATCH 1/3] doi: use a session to ensure that cookies stay set When following the URL chain to arrive at the bibtex format, it can happen that a cookie that was set previously is not sent with the next request. By using an explicit session, we ensure that any cookies are correctly sent for subsequent requests. --- src/cobib/parsers/doi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cobib/parsers/doi.py b/src/cobib/parsers/doi.py index edfaca0d..570b227c 100644 --- a/src/cobib/parsers/doi.py +++ b/src/cobib/parsers/doi.py @@ -69,19 +69,20 @@ class DOIParser(Parser): doi = match.group(1) LOGGER.info("Gathering BibTex data for DOI: %s.", doi) try: - page = requests.get(DOI_URL + doi, headers=DOI_HEADER, timeout=10) + s = requests.Session() + page = s.get(DOI_URL + doi, headers=DOI_HEADER, timeout=10) if page.encoding is None: page.encoding = "utf-8" # this assumes that the doi.org page redirects to the correct journal's landing page redirected_url: str = "" - header = requests.head(DOI_URL + doi, timeout=1).headers + header = s.head(DOI_URL + doi, timeout=1).headers LOGGER.debug("The DOI URL header: '%s'", header) max_iter = 3 while "Location" in header and max_iter: max_iter -= 1 redirected_url = header["Location"] LOGGER.debug("The found URL redirects to: '%s'", redirected_url) - header = requests.head(redirected_url, timeout=1).headers + header = s.head(redirected_url, timeout=1).headers except requests.exceptions.RequestException as err: LOGGER.error("An Exception occurred while trying to query the DOI: %s.", doi) LOGGER.error(err) -- GitLab From 0e72ddd339605608b4eb099d6b9e1e88aec593b2 Mon Sep 17 00:00:00 2001 From: Max Rossmannek Date: Sat, 8 Feb 2025 21:35:08 +0100 Subject: [PATCH 2/3] refactor: propagate use of requests.Session --- src/cobib/importers/zotero.py | 3 ++- src/cobib/parsers/arxiv.py | 3 ++- src/cobib/parsers/doi.py | 8 ++++---- src/cobib/parsers/isbn.py | 3 ++- src/cobib/parsers/url.py | 3 ++- src/cobib/utils/file_downloader.py | 3 ++- tests/parsers/test_arxiv.py | 2 +- tests/parsers/test_doi.py | 2 +- tests/parsers/test_isbn.py | 2 +- tests/utils/test_file_downloader.py | 2 +- 10 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/cobib/importers/zotero.py b/src/cobib/importers/zotero.py index d2bafc8f..ed531ba7 100644 --- a/src/cobib/importers/zotero.py +++ b/src/cobib/importers/zotero.py @@ -144,7 +144,8 @@ class ZoteroImporter(Importer): Event.PreZoteroImport.fire(self) - raw_result = requests.get(self.protected_url, headers=self.authentication, timeout=30) + session = requests.Session() + raw_result = session.get(self.protected_url, headers=self.authentication, timeout=30) if raw_result.encoding is None: raw_result.encoding = "utf-8" diff --git a/src/cobib/parsers/arxiv.py b/src/cobib/parsers/arxiv.py index 397bd588..d15423c2 100644 --- a/src/cobib/parsers/arxiv.py +++ b/src/cobib/parsers/arxiv.py @@ -61,7 +61,8 @@ class ArxivParser(Parser): arxiv_id = match.group(1) LOGGER.info("Gathering BibTex data for arXiv ID: %s.", arxiv_id) try: - page = requests.get(ARXIV_URL + arxiv_id, timeout=10) + session = requests.Session() + page = session.get(ARXIV_URL + arxiv_id, timeout=10) if page.encoding is None: page.encoding = "utf-8" except requests.exceptions.RequestException as err: diff --git a/src/cobib/parsers/doi.py b/src/cobib/parsers/doi.py index 570b227c..ddbd3194 100644 --- a/src/cobib/parsers/doi.py +++ b/src/cobib/parsers/doi.py @@ -69,20 +69,20 @@ class DOIParser(Parser): doi = match.group(1) LOGGER.info("Gathering BibTex data for DOI: %s.", doi) try: - s = requests.Session() - page = s.get(DOI_URL + doi, headers=DOI_HEADER, timeout=10) + session = requests.Session() + page = session.get(DOI_URL + doi, headers=DOI_HEADER, timeout=10) if page.encoding is None: page.encoding = "utf-8" # this assumes that the doi.org page redirects to the correct journal's landing page redirected_url: str = "" - header = s.head(DOI_URL + doi, timeout=1).headers + header = session.head(DOI_URL + doi, timeout=1).headers LOGGER.debug("The DOI URL header: '%s'", header) max_iter = 3 while "Location" in header and max_iter: max_iter -= 1 redirected_url = header["Location"] LOGGER.debug("The found URL redirects to: '%s'", redirected_url) - header = s.head(redirected_url, timeout=1).headers + header = session.head(redirected_url, timeout=1).headers except requests.exceptions.RequestException as err: LOGGER.error("An Exception occurred while trying to query the DOI: %s.", doi) LOGGER.error(err) diff --git a/src/cobib/parsers/isbn.py b/src/cobib/parsers/isbn.py index fb996441..e0ba7592 100644 --- a/src/cobib/parsers/isbn.py +++ b/src/cobib/parsers/isbn.py @@ -64,7 +64,8 @@ class ISBNParser(Parser): LOGGER.info("Gathering BibTex data for ISBN: %s.", isbn) isbn_plain = "".join([i for i in isbn if i.isdigit()]) try: - page = requests.get(ISBN_URL + isbn_plain + "&jscmd=data&format=json", timeout=10) + session = requests.Session() + page = session.get(ISBN_URL + isbn_plain + "&jscmd=data&format=json", timeout=10) if page.encoding is None: page.encoding = "utf-8" except requests.exceptions.RequestException as err: diff --git a/src/cobib/parsers/url.py b/src/cobib/parsers/url.py index cd56ac68..b7c3d05d 100644 --- a/src/cobib/parsers/url.py +++ b/src/cobib/parsers/url.py @@ -63,7 +63,8 @@ class URLParser(Parser): return entries try: - page = requests.get(string, timeout=10) + session = requests.Session() + page = session.get(string, timeout=10) if page.encoding is None: page.encoding = "utf-8" except requests.exceptions.RequestException as err: diff --git a/src/cobib/utils/file_downloader.py b/src/cobib/utils/file_downloader.py index 2b13a38d..53e14cfd 100644 --- a/src/cobib/utils/file_downloader.py +++ b/src/cobib/utils/file_downloader.py @@ -117,7 +117,8 @@ class FileDownloader: LOGGER.info("Downloading %s to %s", url, path) try: - response = requests.get(url, timeout=10, stream=True, headers=headers) + session = requests.Session() + response = session.get(url, timeout=10, stream=True, headers=headers) total_length_str = response.headers.get("content-length", None) total_length = int(total_length_str) if total_length_str is not None else None except requests.exceptions.RequestException as err: diff --git a/tests/parsers/test_arxiv.py b/tests/parsers/test_arxiv.py index a3559512..4d137843 100644 --- a/tests/parsers/test_arxiv.py +++ b/tests/parsers/test_arxiv.py @@ -101,7 +101,7 @@ class TestArxivParser(ParserTest): """Mock function to raise an Exception.""" raise requests.exceptions.RequestException() - monkeypatch.setattr(requests, "get", raise_exception) + monkeypatch.setattr(requests.Session, "get", raise_exception) ArxivParser().parse("1701.0821") assert ( diff --git a/tests/parsers/test_doi.py b/tests/parsers/test_doi.py index d8240385..4c88998d 100644 --- a/tests/parsers/test_doi.py +++ b/tests/parsers/test_doi.py @@ -89,7 +89,7 @@ class TestDOIParser(ParserTest): """Mock function to raise an Exception.""" raise requests.exceptions.RequestException() - monkeypatch.setattr(requests, "get", raise_exception) + monkeypatch.setattr(requests.Session, "get", raise_exception) DOIParser().parse("10.1021/acs.jpclett.3c00330") assert ( diff --git a/tests/parsers/test_isbn.py b/tests/parsers/test_isbn.py index 92d5de15..c6ade696 100644 --- a/tests/parsers/test_isbn.py +++ b/tests/parsers/test_isbn.py @@ -86,7 +86,7 @@ class TestISBNParser(ParserTest): """Mock function to raise an Exception.""" raise requests.exceptions.RequestException() - monkeypatch.setattr(requests, "get", raise_exception) + monkeypatch.setattr(requests.Session, "get", raise_exception) ISBNParser().parse("978-1-449-35573-9") if any( diff --git a/tests/utils/test_file_downloader.py b/tests/utils/test_file_downloader.py index 4f3ed11c..9b353296 100644 --- a/tests/utils/test_file_downloader.py +++ b/tests/utils/test_file_downloader.py @@ -178,7 +178,7 @@ async def test_gracefully_fail_download(monkeypatch: pytest.MonkeyPatch) -> None """Mock function to raise an Exception.""" raise requests.exceptions.RequestException() - monkeypatch.setattr(requests, "get", raise_exception) + monkeypatch.setattr(requests.Session, "get", raise_exception) with tempfile.TemporaryDirectory() as tmpdirname: assert ( -- GitLab From 464c4043537712442703e8a4e54edbde81e1a10f Mon Sep 17 00:00:00 2001 From: Max Rossmannek Date: Sat, 8 Feb 2025 21:41:26 +0100 Subject: [PATCH 3/3] docs: update Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c074c40..d8e71779 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - a hint in the documentation how to filter for (non-)existence of fields (#141) - the coBib command `prompt` to the TUI's command palette (!310) +### Changed +- HTTP requests are now done inside of `requests.Session` instances (!233) + ### Fixed - the `Escape` key during an input prompt triggered after a `Confirm` prompt (e.g. a cancelled `quit`) - the behavior of the `Home` and `End` keybindings in the TUI's search view (!310) -- GitLab