removed unnecessary methods and added a test to the arxiv request

Pipfile

@@ -21,6 +21,7 @@ requests = "*"
 [dev-packages]
 pylint = "*"
 black = "*"
+responses = "*"
 
 [requires]
 python_version="3.6"

tests/use_case/test_arxiv_repo.py

@@ -2,6 +2,7 @@
 """
 
 import pytest
+import responses
 
 from webminer.entities.arxiv_document import ArxivDocument
 from webminer.use_cases.request_arxiv import arxiv_repo as a_repo
@@ -46,6 +47,38 @@
  pdf_url="pfg_url4",
 )
 
+arxiv_result = [
+ {
+ "publish_date": "2018-09-20T17:48:27Z",
+ "authors": ["Mart Kartašev", "Carlo Rapisarda", "Dominik Fay"],
+ "title": "Implementing Adaptive Separable Convolution for Video Frame\n Interpolation",
+ "abstract": "As Deep Neural Networks are becoming more popular, much of the attention is\nbeing devoted to Computer Vision problems that used to be solved with more\ntraditional approaches. Video frame interpolation is one of such challenges\nthat has seen new research involving various techniques in deep learning. In\nthis paper, we replicate the work of Niklaus et al. on Adaptive Separable\nConvolution, which claims high quality results on the video frame interpolation\ntask. We apply the same network structure trained on a smaller dataset and\nexperiment with various different loss functions, in order to determine the\noptimal approach in data-scarce scenarios. The best resulting model is still\nable to provide visually pleasing videos, although achieving lower evaluation\nscores.",
+ "id": "http:https://arxiv.org/abs/1809.07759v1",
+ "link": "http:https://arxiv.org/abs/1809.07759v1",
+ "pdf": "http:https://arxiv.org/pdf/1809.07759v1",
+ "_rawid": "1809.07759",
+ "_version": 1,
+ },
+ {
+ "publish_date": "2018-06-05T03:49:46Z",
+ "authors": [
+ "Amir Sadeghian",
+ "Vineet Kosaraju",
+ "Ali Sadeghian",
+ "Noriaki Hirose",
+ "S. Hamid Rezatofighi",
+ "Silvio Savarese",
+ ],
+ "title": "SoPhie: An Attentive GAN for Predicting Paths Compliant to Social and\n Physical Constraints",
+ "abstract": "This paper addresses the problem of path prediction for multiple interacting\nagents in a scene, which is a crucial step for many autonomous platforms such\nas self-driving cars and social robots. We present \textit{SoPhie}; an\ninterpretable framework based on Generative Adversarial Network (GAN), which\nleverages two sources of information, the path history of all the agents in a\nscene, and the scene context information, using images of the scene. To predict\na future path for an agent, both physical and social information must be\nleveraged. Previous work has not been successful to jointly model physical and\nsocial interactions. Our approach blends a social attention mechanism with a\nphysical attention that helps the model to learn where to look in a large scene\nand extract the most salient parts of the image relevant to the path. Whereas,\nthe social attention component aggregates information across the different\nagent interactions and extracts the most important trajectory information from\nthe surrounding neighbors. SoPhie also takes advantage of GAN to generates more\nrealistic samples and to capture the uncertain nature of the future paths by\nmodeling its distribution. All these mechanisms enable our approach to predict\nsocially and physically plausible paths for the agents and to achieve\nstate-of-the-art performance on several different trajectory forecasting\nbenchmarks.",
+ "id": "http:https://arxiv.org/abs/1806.01482v2",
+ "link": "http:https://arxiv.org/abs/1806.01482v2",
+ "pdf": "http:https://arxiv.org/pdf/1806.01482v2",
+ "_rawid": "1806.01482",
+ "_version": 2,
+ },
+]
+
 
 @pytest.fixture
 def domain_arxivdocs():
@@ -73,3 +106,110 @@ def test_repository_list_without_parameters(domain_arxivdocs):
  repo = a_repo.ArxivRepo(domain_arxivdocs)
 
  assert_equal(repo.list(), domain_arxivdocs)
+
+
+@responses.activate
+def test_extract_relevant_info():
+ url = "http:https://export.arxiv.org/api/query?search_query=cat:cs.CV+OR+cat:cs.AI+OR+cat:cs.LG+OR+cat:cs.CL+OR+cat:cs.NE+OR+cat:stat.ML&sortBy=lastUpdatedDate&start=0&max_results=100"
+ responses.add(
+ method=responses.GET,
+ url=url,
+ body="""<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http:https://www.w3.org/2005/Atom">
+ <link href="http:https://arxiv.org/api/query?search_query%3Dcat%3Acs.CV%20OR%20cat%3Acs.AI%20OR%20cat%3Acs.LG%20OR%20cat%3Acs.CL%20OR%20cat%3Acs.NE%20OR%20cat%3Astat.ML%26id_list%3D%26start%3D1%26max_results%3D100" rel="self" type="application/atom+xml"/>
+ <title type="html">ArXiv Query: search_query=cat:cs.CV OR cat:cs.AI OR cat:cs.LG OR cat:cs.CL OR cat:cs.NE OR cat:stat.ML&amp;id_list=&amp;start=1&amp;max_results=100</title>
+ <id>http:https://arxiv.org/api//mx7Y+oW1RP05QqmCZfHNto2duM</id>
+ <updated>2018-09-23T00:00:00-04:00</updated>
+ <opensearch:totalResults xmlns:opensearch="http:https://a9.com/-/spec/opensearch/1.1/">68000</opensearch:totalResults>
+ <opensearch:startIndex xmlns:opensearch="http:https://a9.com/-/spec/opensearch/1.1/">1</opensearch:startIndex>
+ <opensearch:itemsPerPage xmlns:opensearch="http:https://a9.com/-/spec/opensearch/1.1/">100</opensearch:itemsPerPage>
+ <entry>
+ <id>http:https://arxiv.org/abs/1809.07759v1</id>
+ <updated>2018-09-20T17:48:27Z</updated>
+ <published>2018-09-20T17:48:27Z</published>
+ <title>Implementing Adaptive Separable Convolution for Video Frame
+ Interpolation</title>
+ <summary> As Deep Neural Networks are becoming more popular, much of the attention is
+being devoted to Computer Vision problems that used to be solved with more
+traditional approaches. Video frame interpolation is one of such challenges
+that has seen new research involving various techniques in deep learning. In
+this paper, we replicate the work of Niklaus et al. on Adaptive Separable
+Convolution, which claims high quality results on the video frame interpolation
+task. We apply the same network structure trained on a smaller dataset and
+experiment with various different loss functions, in order to determine the
+optimal approach in data-scarce scenarios. The best resulting model is still
+able to provide visually pleasing videos, although achieving lower evaluation
+scores.
+</summary>
+ <author>
+ <name>Mart Kartašev</name>
+ </author>
+ <author>
+ <name>Carlo Rapisarda</name>
+ </author>
+ <author>
+ <name>Dominik Fay</name>
+ </author>
+ <arxiv:comment xmlns:arxiv="http:https://arxiv.org/schemas/atom">All authors contributed equally</arxiv:comment>
+ <link href="http:https://arxiv.org/abs/1809.07759v1" rel="alternate" type="text/html"/>
+ <link title="pdf" href="http:https://arxiv.org/pdf/1809.07759v1" rel="related" type="application/pdf"/>
+ <arxiv:primary_category xmlns:arxiv="http:https://arxiv.org/schemas/atom" term="cs.CV" scheme="http:https://arxiv.org/schemas/atom"/>
+ <category term="cs.CV" scheme="http:https://arxiv.org/schemas/atom"/>
+ </entry>
+ <entry>
+ <id>http:https://arxiv.org/abs/1806.01482v2</id>
+ <updated>2018-09-20T17:42:42Z</updated>
+ <published>2018-06-05T03:49:46Z</published>
+ <title>SoPhie: An Attentive GAN for Predicting Paths Compliant to Social and
+ Physical Constraints</title>
+ <summary> This paper addresses the problem of path prediction for multiple interacting
+agents in a scene, which is a crucial step for many autonomous platforms such
+as self-driving cars and social robots. We present \textit{SoPhie}; an
+interpretable framework based on Generative Adversarial Network (GAN), which
+leverages two sources of information, the path history of all the agents in a
+scene, and the scene context information, using images of the scene. To predict
+a future path for an agent, both physical and social information must be
+leveraged. Previous work has not been successful to jointly model physical and
+social interactions. Our approach blends a social attention mechanism with a
+physical attention that helps the model to learn where to look in a large scene
+and extract the most salient parts of the image relevant to the path. Whereas,
+the social attention component aggregates information across the different
+agent interactions and extracts the most important trajectory information from
+the surrounding neighbors. SoPhie also takes advantage of GAN to generates more
+realistic samples and to capture the uncertain nature of the future paths by
+modeling its distribution. All these mechanisms enable our approach to predict
+socially and physically plausible paths for the agents and to achieve
+state-of-the-art performance on several different trajectory forecasting
+benchmarks.
+</summary>
+ <author>
+ <name>Amir Sadeghian</name>
+ </author>
+ <author>
+ <name>Vineet Kosaraju</name>
+ </author>
+ <author>
+ <name>Ali Sadeghian</name>
+ </author>
+ <author>
+ <name>Noriaki Hirose</name>
+ </author>
+ <author>
+ <name>S. Hamid Rezatofighi</name>
+ </author>
+ <author>
+ <name>Silvio Savarese</name>
+ </author>
+ <link href="http:https://arxiv.org/abs/1806.01482v2" rel="alternate" type="text/html"/>
+ <link title="pdf" href="http:https://arxiv.org/pdf/1806.01482v2" rel="related" type="application/pdf"/>
+ <arxiv:primary_category xmlns:arxiv="http:https://arxiv.org/schemas/atom" term="cs.CV" scheme="http:https://arxiv.org/schemas/atom"/>
+ <category term="cs.CV" scheme="http:https://arxiv.org/schemas/atom"/>
+ </entry>""",
+ status=200,
+ )
+
+ repo = a_repo.ArxivRepo()
+ result = repo.fetch_papers()
+
+ assert len(result) > 0
+ assert result == arxiv_result

webminer/use_cases/request_arxiv/arxiv_repo.py

@@ -90,24 +90,26 @@ def list(self, filters=None):
 
  return [ad.ArxivDocument.from_dict(r) for r in result]
 
- def encode_feedparser_dict(self, fp_dict):
- """
- recursive function to convert the internal feedparse object to a simple dict
- """
- if isinstance(fp_dict, feedparser.FeedParserDict) or isinstance(fp_dict, dict):
- ret_dict = {}
- for key in fp_dict.keys():
- ret_dict[key] = self.encode_feedparser_dict(fp_dict[key])
- return ret_dict
- elif isinstance(fp_dict, list):
- dict_list = []
- for key in fp_dict:
- dict_list.append(self.encode_feedparser_dict(key))
- return dict_list
- else:
- return fp_dict
-
- def extract_relevant_info(self, fp_dict):
+ # @classmethod
+ # def encode_feedparser_dict(cls, fp_dict):
+ # """
+ # recursive function to convert the internal feedparse object to a simple dict
+ # """
+ # if isinstance(fp_dict, feedparser.FeedParserDict) or isinstance(fp_dict, dict):
+ # ret_dict = {}
+ # for key in fp_dict.keys():
+ # ret_dict[key] = self.encode_feedparser_dict(fp_dict[key])
+ # return ret_dict
+ # elif isinstance(fp_dict, list):
+ # dict_list = []
+ # for key in fp_dict:
+ # dict_list.append(self.encode_feedparser_dict(key))
+ # return dict_list
+ # else:
+ # return fp_dict
+
+ @classmethod
+ def extract_relevant_info(cls, fp_dict):
  """Extracts the relevant info
 
  Args:
@@ -149,11 +151,12 @@ def run_query(self, search_query, start=0, max_results=10):
 
  query = f"search_query={search_query}&sortBy=lastUpdatedDate&start={start}&max_results={max_results}" # pylint: disable=C0301
  r = requests.get(self.base_url + query)
- parsed_response = feedparser.parse(r.text())
+ parsed_response = feedparser.parse(r.text)
 
  return parsed_response
 
- def parse_arxiv_url(self, url):
+ @classmethod
+ def parse_arxiv_url(cls, url):
  """
  extracts the raw id and the version
  examples is http:https://arxiv.org/abs/1512.08756v2