# News Application using Neurelo’s Python SDKs

## <mark style="color:green;">**`feeds.json`**</mark>

```json
{
  "trending": [
    "https://feeds.bbci.co.uk/news/rss.xml",
    "https://www.cbc.ca/webfeed/rss/rss-topstories"
  ]
}
```

## <mark style="color:green;">**`fetch.py`**</mark>

```python
import feedparser
import scrapy
from scrapy.crawler import CrawlerProcess
import json
from newspaper import fulltext
import requests
from dotenv import load_dotenv
import os
import argparse

from neurelo.configuration import Configuration
from neurelo.api_client import ApiClient
from neurelo.api.news_api import NewsApi

scraped = {}

def start_scrapy(urls):
    process = CrawlerProcess(settings={"FEED_FORMAT": "json", "FEED_URI": "items.json"})
    process.crawl(Scrape, kwargs={"urls": urls})
    process.start()

class Scrape(scrapy.Spider):
    def __init__(self, **kwargs):
        self.name = ""
        self.urls = kwargs["kwargs"]["urls"]

    def start_requests(self):
        for url in self.urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        try:
            scraped[response.request.meta["redirect_urls"][0]] = response.text
        except:
            scraped[response.url] = response.text

class Parse:
    def __init__(self):
        pass

    def fetch_feeds(self):
        feeds = json.load(open("./feeds.json"))
        topics = list(feeds.keys())
        return (feeds, topics)

    def parse_rss_feeds(self, feeds, topics, limit=5):
        content = {}
        need_to_scrape = []
        index = 0
        for topic in topics:
            for feed in feeds[topic]:
                fetched = feedparser.parse(feed)
                for news in fetched.entries[:limit]:
                    content[index] = {}
                    content[index]["title"] = news["title"]
                    content[index]["link"] = news["link"]
                    content[index]["published"] = news["published"]
                    content[index]["category"] = topic
                    need_to_scrape.append(news["link"])
                    index += 1
        return (content, need_to_scrape)

    def get_news(self, rss_content):
        for index in rss_content.keys():
            feed_url = rss_content[index]["link"]
            scraped_news = fulltext(scraped[feed_url], language="en")
            rss_content[index]["summary"] = scraped_news
        return rss_content

    def parse(self):
        rss_feeds, topics = self.fetch_feeds()
        content, need_to_scrape = self.parse_rss_feeds(rss_feeds, topics)
        start_scrapy(need_to_scrape)
        return self.get_news(content)

class News:
    def __init__(self) -> None:
        host, key = self.env()
        self.api_client = self.conf(host, key)

    def env(self):
        load_dotenv()
        return (os.getenv("NEURELO_API_HOST") or "", os.getenv("NEURELO_API_KEY") or "")

    def conf(self, host, key):
        configuration = Configuration(host, api_key={"ApiKey": key})
        return ApiClient(configuration=configuration)

    def fetch(self):
        news_api = NewsApi(self.api_client)
        news_item = news_api.find_news()
        return news_item

    def store(self):
        parse = Parse()
        news = parse.parse()
        articles = list(news.values())

        news_api = NewsApi(self.api_client)
        response = news_api.create_many_news(articles)
        return response

    def delete(self):
        news_api = NewsApi(self.api_client)
        deleted = news_api.delete_news()
        return deleted

if __name__ == "__main__":
    news = News()
    print(news.fetch())
```


---

# Agent Instructions: Querying This Documentation

If you need additional information that is not directly available in this page, you can query the documentation dynamically by asking a question.

Perform an HTTP GET request on the current page URL with the `ask` query parameter:

```
GET https://docs.neurelo.com/sdks/python-sdk/python-sdk-tutorial-news-application/news-application-using-neurelos-python-sdks.md?ask=<question>
```

The question should be specific, self-contained, and written in natural language.
The response will contain a direct answer to the question and relevant excerpts and sources from the documentation.

Use this mechanism when the answer is not explicitly present in the current page, you need clarification or additional context, or you want to retrieve related documentation sections.