diff options
author | Andreas Krennmair <ak@synflood.at> | 2007-08-07 09:11:45 +0000 |
---|---|---|
committer | Andreas Krennmair <ak@synflood.at> | 2007-08-07 09:11:45 +0000 |
commit | 1f873eedf8b81a2b5cfc4031a1d77578460de072 (patch) | |
tree | 117d7e2d1f127d56b99e03160644629e728a5a5d | |
parent | 58750b4617fd97e971e7e650a2da755afd5698ae (diff) |
Andreas Krennmair:
implemented the new "query feeds" feature.
-rw-r--r-- | doc/newsbeuter.txt | 31 | ||||
-rw-r--r-- | include/controller.h | 2 | ||||
-rw-r--r-- | include/rss.h | 10 | ||||
-rw-r--r-- | src/cache.cpp | 3 | ||||
-rw-r--r-- | src/controller.cpp | 2 | ||||
-rw-r--r-- | src/itemview_formaction.cpp | 14 | ||||
-rw-r--r-- | src/rss.cpp | 267 | ||||
-rw-r--r-- | src/view.cpp | 1 |
8 files changed, 216 insertions, 114 deletions
diff --git a/doc/newsbeuter.txt b/doc/newsbeuter.txt index e4bb144b..b132c481 100644 --- a/doc/newsbeuter.txt +++ b/doc/newsbeuter.txt @@ -463,6 +463,37 @@ presented to the user. The configuration itself can contain as many ignore-article commands as desired. +Query Feeds +~~~~~~~~~~~ + +Query feeds are a mechanism of newsbeuter to define custom "meta feeds" by using +newsbeuter's built-in filter language. A query feed is a feed that is aggregated +from all currently downloaded articles of all feeds. To narrow down the set of +articles, the user has to specify a filter. Only articles that match this filter +are added to the query feed. A query feed is updated whenever it is entered in +the feed list. When you change the unread flag of an article, this is reflected +in the feed where the article was originally fetched. + +To define a query feed, the user has to add a line to the file +~/.newsbeuter/urls in the following format: + + query:<name of feed>:<filter expression> [<tag> ...] + +The "query:" in the beginning tells newsbeuter that it's a query feed, "<name of +feed>" specifies the name under which the query feed shall be displayed in the +feed list, and "<filter expression>" is the filter expression that shall be +used. Like every other feed, a query feed can be tagged to organize it like +a regular feed. + +A good example for the user of this feature is a query feed that contains all +unread articles: + + "query:Unread Articles:unread = \"yes\"" + +Note the quotes that are necessary around the complete query "URL" and the +backslashes that are necessary the escape the quotes in the filter expression. + + Podcast Support ~~~~~~~~~~~~~~~ diff --git a/include/controller.h b/include/controller.h index 0a9bbb88..2ef87cd7 100644 --- a/include/controller.h +++ b/include/controller.h @@ -41,6 +41,8 @@ namespace newsbeuter { void reload_urls_file(); + inline std::vector<rss_feed>& get_all_feeds() { return feeds; } + private: void usage(char * argv0); void import_opml(const char * filename); diff --git a/include/rss.h b/include/rss.h index e4dfd4ea..f23d245b 100644 --- a/include/rss.h +++ b/include/rss.h @@ -43,6 +43,8 @@ namespace newsbeuter { return pubDate_; } void set_pubDate(time_t t); + + bool operator<(const rss_item& item) const { return item.pubDate_ < this->pubDate_; } // new items come first inline const std::string& guid() const { return guid_; } void set_guid(const std::string& g); @@ -50,6 +52,7 @@ namespace newsbeuter { inline bool unread() const { return unread_; } void set_unread(bool u); void set_unread_nowrite(bool u); + void set_unread_nowrite_notify(bool u); inline void set_cache(cache * c) { ch = c; } inline void set_feedurl(const std::string& f) { feedurl_ = f; } @@ -110,7 +113,7 @@ namespace newsbeuter { rss_item& get_item_by_guid(const std::string& guid); inline const std::string& rssurl() const { return rssurl_; } - inline void set_rssurl(const std::string& u) { rssurl_ = u; } + void set_rssurl(const std::string& u); unsigned int unread_item_count() const; @@ -121,6 +124,10 @@ namespace newsbeuter { virtual bool has_attribute(const std::string& attribname); virtual std::string get_attribute(const std::string& attribname); + void update_items(std::vector<rss_feed>& feeds); + + inline void set_query(const std::string& s) { query = s; } + private: std::string title_; std::string description_; @@ -129,6 +136,7 @@ namespace newsbeuter { std::string rssurl_; std::vector<rss_item> items_; std::vector<std::string> tags_; + std::string query; cache * ch; }; diff --git a/src/cache.cpp b/src/cache.cpp index 5cdcbc8d..f92b913c 100644 --- a/src/cache.cpp +++ b/src/cache.cpp @@ -235,6 +235,9 @@ void cache::populate_tables() { // this function writes an rss_feed including all rss_items to the database void cache::externalize_rssfeed(rss_feed& feed) { + if (feed.rssurl().substr(0,6) == "query:") + return; + mtx->lock(); std::ostringstream query; query << "SELECT count(*) FROM rss_feed WHERE rssurl = '" << feed.rssurl() << "';"; diff --git a/src/controller.cpp b/src/controller.cpp index 2bd516ec..9ae9aa4a 100644 --- a/src/controller.cpp +++ b/src/controller.cpp @@ -335,7 +335,7 @@ void controller::mark_all_read(unsigned int pos) { rsscache->catchup_all(feed.rssurl()); if (feed.items().size() > 0) { for (std::vector<rss_item>::iterator it=feed.items().begin();it!=feed.items().end();++it) { - it->set_unread_nowrite(false); + it->set_unread_nowrite_notify(false); } } } diff --git a/src/itemview_formaction.cpp b/src/itemview_formaction.cpp index 43d58b59..38c712bc 100644 --- a/src/itemview_formaction.cpp +++ b/src/itemview_formaction.cpp @@ -30,15 +30,17 @@ void itemview_formaction::prepare() { rss_item& item = feed->get_item_by_guid(guid); std::string code = "{list"; + rss_feed * feedptr = item.get_feedptr(); + code.append("{listitem text:"); std::ostringstream feedtitle; feedtitle << _("Feed: "); - if (feed->title().length() > 0) { - feedtitle << feed->title(); - } else if (feed->link().length() > 0) { - feedtitle << feed->link(); - } else if (feed->rssurl().length() > 0) { - feedtitle << feed->rssurl(); + if (feedptr->title().length() > 0) { + feedtitle << feedptr->title(); + } else if (feedptr->link().length() > 0) { + feedtitle << feedptr->link(); + } else if (feedptr->rssurl().length() > 0) { + feedtitle << feedptr->rssurl(); } code.append(stfl::quote(feedtitle.str().c_str())); code.append("}"); diff --git a/src/rss.cpp b/src/rss.cpp index dd090936..aae67f3d 100644 --- a/src/rss.cpp +++ b/src/rss.cpp @@ -24,6 +24,7 @@ rss_parser::~rss_parser() { } rss_feed rss_parser::parse() { rss_feed feed(ch); + bool skip_parsing = false; feed.set_rssurl(my_uri); @@ -62,140 +63,148 @@ rss_feed rss_parser::parse() { std::string result = utils::run_filter(filter, buf); GetLogger().log(LOG_DEBUG, "rss_parser::parse: output of `%s' is: %s", filter.c_str(), result.c_str()); err = mrss_parse_buffer(const_cast<char *>(result.c_str()), result.length(), &mrss); + } else if (my_uri.substr(0,6) == "query:") { + + skip_parsing = true; + } else { char buf[1024]; snprintf(buf, sizeof(buf), _("Error: unsupported URL: %s"), my_uri.c_str()); throw std::string(buf); } - if (err != MRSS_OK) { - if (err == MRSS_ERR_POSIX) { - GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed with POSIX error: error = %s",strerror(errno)); - } - GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed: err = %s (%d)",mrss_strerror(err), err); - GetLogger().log(LOG_USERERROR, "RSS feed `%s' couldn't be parsed: %s (error %d)", my_uri.c_str(), mrss_strerror(err), err); - if (mrss) { - mrss_free(mrss); + if (!skip_parsing) { + + if (err != MRSS_OK) { + if (err == MRSS_ERR_POSIX) { + GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed with POSIX error: error = %s",strerror(errno)); + } + GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed: err = %s (%d)",mrss_strerror(err), err); + GetLogger().log(LOG_USERERROR, "RSS feed `%s' couldn't be parsed: %s (error %d)", my_uri.c_str(), mrss_strerror(err), err); + if (mrss) { + mrss_free(mrss); + } + throw std::string(mrss_strerror(err)); } - throw std::string(mrss_strerror(err)); - } - const char * encoding = mrss->encoding ? mrss->encoding : "utf-8"; + const char * encoding = mrss->encoding ? mrss->encoding : "utf-8"; - if (mrss->title) { - feed.set_title(utils::convert_text(mrss->title, "utf-8", encoding)); - } - - if (mrss->description) { - feed.set_description(utils::convert_text(mrss->description, "utf-8", encoding)); - } + if (mrss->title) { + feed.set_title(utils::convert_text(mrss->title, "utf-8", encoding)); + } + + if (mrss->description) { + feed.set_description(utils::convert_text(mrss->description, "utf-8", encoding)); + } - if (mrss->link) feed.set_link(mrss->link); - if (mrss->pubDate) - feed.set_pubDate(parse_date(mrss->pubDate)); - else - feed.set_pubDate(::time(NULL)); + if (mrss->link) feed.set_link(mrss->link); + if (mrss->pubDate) + feed.set_pubDate(parse_date(mrss->pubDate)); + else + feed.set_pubDate(::time(NULL)); - GetLogger().log(LOG_DEBUG, "rss_parser::parse: feed title = `%s' link = `%s'", feed.title().c_str(), feed.link().c_str()); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: feed title = `%s' link = `%s'", feed.title().c_str(), feed.link().c_str()); - for (mrss_item_t * item = mrss->item; item != NULL; item = item->next ) { - rss_item x(ch); - if (item->title) { - std::string title = utils::convert_text(item->title, "utf-8", encoding); - x.set_title(title); - GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted title `%s' to `%s'", item->title, title.c_str()); - } - if (item->link) x.set_link(item->link); - if (item->author) { - x.set_author(utils::convert_text(item->author, "utf-8", encoding)); - } + for (mrss_item_t * item = mrss->item; item != NULL; item = item->next ) { + rss_item x(ch); + if (item->title) { + std::string title = utils::convert_text(item->title, "utf-8", encoding); + x.set_title(title); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted title `%s' to `%s'", item->title, title.c_str()); + } + if (item->link) x.set_link(item->link); + if (item->author) { + x.set_author(utils::convert_text(item->author, "utf-8", encoding)); + } - x.set_feedurl(feed.rssurl()); + x.set_feedurl(feed.rssurl()); - mrss_tag_t * content; + mrss_tag_t * content; - if (mrss_search_tag(item, "encoded", "http://purl.org/rss/1.0/modules/content/", &content) == MRSS_OK && content) { - /* RSS 2.0 content:encoded */ - GetLogger().log(LOG_DEBUG, "rss_parser::parse: found content:encoded: %s\n", content->value); - if (content->value) { - std::string desc = utils::convert_text(content->value, "utf-8", encoding); - GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted description `%s' to `%s'", content->value, desc.c_str()); - x.set_description(desc); + if (mrss_search_tag(item, "encoded", "http://purl.org/rss/1.0/modules/content/", &content) == MRSS_OK && content) { + /* RSS 2.0 content:encoded */ + GetLogger().log(LOG_DEBUG, "rss_parser::parse: found content:encoded: %s\n", content->value); + if (content->value) { + std::string desc = utils::convert_text(content->value, "utf-8", encoding); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted description `%s' to `%s'", content->value, desc.c_str()); + x.set_description(desc); + } + } else { + GetLogger().log(LOG_DEBUG, "rss_parser::parse: found no content:encoded"); } - } else { - GetLogger().log(LOG_DEBUG, "rss_parser::parse: found no content:encoded"); - } - if ((mrss->version == MRSS_VERSION_ATOM_0_3 || mrss->version == MRSS_VERSION_ATOM_1_0)) { - int rc; - if (((rc = mrss_search_tag(item, "content", "http://www.w3.org/2005/Atom", &content)) == MRSS_OK && content) || - ((rc = mrss_search_tag(item, "content", "http://purl.org/atom/ns#", &content)) == MRSS_OK && content)) { - GetLogger().log(LOG_DEBUG, "rss_parser::parse: found atom content: %s\n", content ? content->value : "(content = null)"); - if (content && content->value) { - x.set_description(utils::convert_text(content->value, "utf-8", encoding)); + if ((mrss->version == MRSS_VERSION_ATOM_0_3 || mrss->version == MRSS_VERSION_ATOM_1_0)) { + int rc; + if (((rc = mrss_search_tag(item, "content", "http://www.w3.org/2005/Atom", &content)) == MRSS_OK && content) || + ((rc = mrss_search_tag(item, "content", "http://purl.org/atom/ns#", &content)) == MRSS_OK && content)) { + GetLogger().log(LOG_DEBUG, "rss_parser::parse: found atom content: %s\n", content ? content->value : "(content = null)"); + if (content && content->value) { + x.set_description(utils::convert_text(content->value, "utf-8", encoding)); + } + } else { + GetLogger().log(LOG_DEBUG, "rss_parser::parse: mrss_search_tag(content) failed with rc = %d content = %p", rc, content); } } else { - GetLogger().log(LOG_DEBUG, "rss_parser::parse: mrss_search_tag(content) failed with rc = %d content = %p", rc, content); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: not an atom feed"); } - } else { - GetLogger().log(LOG_DEBUG, "rss_parser::parse: not an atom feed"); - } - /* last resort: search for itunes:summary tag (may be a podcast) */ - if (x.description().length() == 0 && mrss_search_tag(item, "summary", "http://www.itunes.com/dtds/podcast-1.0.dtd", &content) == MRSS_OK && content) { - GetLogger().log(LOG_DEBUG, "rss_parser::parse: found itunes:summary: %s\n", content->value); - if (content->value) { - std::string desc = "<ituneshack>"; - desc.append(utils::convert_text(content->value, "utf-8", encoding)); - desc.append("</ituneshack>"); - x.set_description(desc); + /* last resort: search for itunes:summary tag (may be a podcast) */ + if (x.description().length() == 0 && mrss_search_tag(item, "summary", "http://www.itunes.com/dtds/podcast-1.0.dtd", &content) == MRSS_OK && content) { + GetLogger().log(LOG_DEBUG, "rss_parser::parse: found itunes:summary: %s\n", content->value); + if (content->value) { + std::string desc = "<ituneshack>"; + desc.append(utils::convert_text(content->value, "utf-8", encoding)); + desc.append("</ituneshack>"); + x.set_description(desc); + } + + } else { + GetLogger().log(LOG_DEBUG, "rss_parser::parse: no luck with itunes:summary"); } - - } else { - GetLogger().log(LOG_DEBUG, "rss_parser::parse: no luck with itunes:summary"); - } - if (x.description().length() == 0 && item->description) { - x.set_description(utils::convert_text(item->description, "utf-8", encoding)); - } + if (x.description().length() == 0 && item->description) { + x.set_description(utils::convert_text(item->description, "utf-8", encoding)); + } - if (item->pubDate) - x.set_pubDate(parse_date(item->pubDate)); - else - x.set_pubDate(::time(NULL)); - - if (item->guid) - x.set_guid(item->guid); - else if (item->link) - x.set_guid(item->link); // XXX hash something to get a better alternative GUID - else if (item->title) - x.set_guid(item->title); - // ...else?! that's too bad. - - if (item->enclosure_url) { - x.set_enclosure_url(item->enclosure_url); - GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_url: %s", item->enclosure_url); - } - if (item->enclosure_type) { - x.set_enclosure_type(item->enclosure_type); - GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_type: %s", item->enclosure_type); - } + if (item->pubDate) + x.set_pubDate(parse_date(item->pubDate)); + else + x.set_pubDate(::time(NULL)); + + if (item->guid) + x.set_guid(item->guid); + else if (item->link) + x.set_guid(item->link); // XXX hash something to get a better alternative GUID + else if (item->title) + x.set_guid(item->title); + // ...else?! that's too bad. + + if (item->enclosure_url) { + x.set_enclosure_url(item->enclosure_url); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_url: %s", item->enclosure_url); + } + if (item->enclosure_type) { + x.set_enclosure_type(item->enclosure_type); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_type: %s", item->enclosure_type); + } - x.set_feedptr(&feed); + x.set_feedptr(&feed); - GetLogger().log(LOG_DEBUG, "rss_parser::parse: item title = `%s' link = `%s' pubDate = `%s' (%d) description = `%s'", - x.title().c_str(), x.link().c_str(), x.pubDate().c_str(), x.pubDate_timestamp(), x.description().c_str()); + GetLogger().log(LOG_DEBUG, "rss_parser::parse: item title = `%s' link = `%s' pubDate = `%s' (%d) description = `%s'", + x.title().c_str(), x.link().c_str(), x.pubDate().c_str(), x.pubDate_timestamp(), x.description().c_str()); - // only add item to feed if it isn't on the ignore list or if there is no ignore list - if (!ign || !ign->matches(&x)) { - feed.items().push_back(x); - GetLogger().log(LOG_INFO, "rss_parser::parse: added article title = `%s' link = `%s' ign = %p", x.title().c_str(), x.link().c_str(), ign); - } else { - GetLogger().log(LOG_INFO, "rss_parser::parse: ignored article title = `%s' link = `%s'", x.title().c_str(), x.link().c_str()); + // only add item to feed if it isn't on the ignore list or if there is no ignore list + if (!ign || !ign->matches(&x)) { + feed.items().push_back(x); + GetLogger().log(LOG_INFO, "rss_parser::parse: added article title = `%s' link = `%s' ign = %p", x.title().c_str(), x.link().c_str(), ign); + } else { + GetLogger().log(LOG_INFO, "rss_parser::parse: ignored article title = `%s' link = `%s'", x.title().c_str(), x.link().c_str()); + } } - } - mrss_free(mrss); + mrss_free(mrss); + + } return feed; } @@ -232,10 +241,18 @@ void rss_item::set_unread_nowrite(bool u) { unread_ = u; } +void rss_item::set_unread_nowrite_notify(bool u) { + unread_ = u; + if (feedptr) + feedptr->get_item_by_guid(guid_).set_unread_nowrite(unread_); // notify parent feed +} + void rss_item::set_unread(bool u) { if (unread_ != u) { bool old_u = unread_; unread_ = u; + if (feedptr) + feedptr->get_item_by_guid(guid_).set_unread_nowrite(unread_); // notify parent feed try { if (ch) ch->update_rssitem_unread_and_enqueued(*this, feedurl_); } catch (const dbexception& e) { @@ -390,7 +407,8 @@ std::string rss_feed::description() const { rss_item& rss_feed::get_item_by_guid(const std::string& guid) { for (std::vector<rss_item>::iterator it=items_.begin();it!=items_.end();++it) { if (it->guid() == guid) { - it->set_feedptr(this); + if (rssurl_.substr(0,6) != "query:") + it->set_feedptr(this); return *it; } } @@ -519,3 +537,40 @@ bool rss_ignores::matches(rss_item* item) { } return false; } + +void rss_feed::update_items(std::vector<rss_feed>& feeds) { + if (query.length() == 0) + return; + + GetLogger().log(LOG_DEBUG, "rss_feed::update_items: query = `%s'", query.c_str()); + + matcher m(query); + + if (items_.size() > 0) { + items_.erase(items_.begin(), items_.end()); + } + + for (std::vector<rss_feed>::iterator it=feeds.begin();it!=feeds.end();++it) { + if (it->rssurl().substr(0,6) != "query:") { // don't fetch items from other query feeds! + for (std::vector<rss_item>::iterator jt=it->items().begin();jt!=it->items().end();++jt) { + if (m.matches(&(*jt))) { + GetLogger().log(LOG_DEBUG, "rss_feed::update_items: matcher matches!"); + jt->set_feedptr(&(*it)); + items_.push_back(*jt); + } + } + } + } + + sort(items_.begin(), items_.end()); +} + +void rss_feed::set_rssurl(const std::string& u) { + rssurl_ = u; + if (rssurl_.substr(0,6) == "query:") { + std::vector<std::string> tokens = utils::tokenize_quoted(u, ":"); + GetLogger().log(LOG_DEBUG, "rss_feed::set_rssurl: query name = `%s' expr = `%s'", tokens[1].c_str(), tokens[2].c_str()); + set_title(tokens[1]); + set_query(tokens[2]); + } +} diff --git a/src/view.cpp b/src/view.cpp index 97cb3ad1..ff3902e2 100644 --- a/src/view.cpp +++ b/src/view.cpp @@ -285,6 +285,7 @@ void view::set_tags(const std::vector<std::string>& t) { void view::push_itemlist(unsigned int pos) { rss_feed * feed = ctrl->get_feed(pos); GetLogger().log(LOG_DEBUG, "view::push_itemlist: retrieved feed at position %d (address = %p)", pos, feed); + feed->update_items(ctrl->get_all_feeds()); if (feed->items().size() > 0) { itemlist->set_feed(feed); itemlist->set_pos(pos); |