Andreas Krennmair:

	implemented the new "query feeds" feature.

8 files changed, 216 insertions, 114 deletions

diff --git a/doc/newsbeuter.txt b/doc/newsbeuter.txt
index e4bb144b..b132c481 100644
--- a/doc/newsbeuter.txt
+++ b/doc/newsbeuter.txt
@@ -463,6 +463,37 @@ presented to the user. The configuration itself can contain as many
 ignore-article commands as desired.
 
 
+Query Feeds
+~~~~~~~~~~~
+
+Query feeds are a mechanism of newsbeuter to define custom "meta feeds" by using 
+newsbeuter's built-in filter language. A query feed is a feed that is aggregated
+from all currently downloaded articles of all feeds. To narrow down the set of
+articles, the user has to specify a filter. Only articles that match this filter
+are added to the query feed. A query feed is updated whenever it is entered in
+the feed list. When you change the unread flag of an article, this is reflected
+in the feed where the article was originally fetched.
+
+To define a query feed, the user has to add a line to the file
+~/.newsbeuter/urls in the following format:
+
+	query:<name of feed>:<filter expression> [<tag> ...]
+
+The "query:" in the beginning tells newsbeuter that it's a query feed, "<name of
+feed>" specifies the name under which the query feed shall be displayed in the
+feed list, and "<filter expression>" is the filter expression that shall be
+used. Like every other feed, a query feed can be tagged to organize it like
+a regular feed.
+
+A good example for the user of this feature is a query feed that contains all
+unread articles:
+
+	"query:Unread Articles:unread = \"yes\""
+
+Note the quotes that are necessary around the complete query "URL" and the
+backslashes that are necessary the escape the quotes in the filter expression.
+
+
 Podcast Support
 ~~~~~~~~~~~~~~~
 
diff --git a/include/controller.h b/include/controller.h
index 0a9bbb88..2ef87cd7 100644
--- a/include/controller.h
+++ b/include/controller.h
@@ -41,6 +41,8 @@ namespace newsbeuter {
 
 			void reload_urls_file();
 
+			inline std::vector<rss_feed>& get_all_feeds() { return feeds; }
+
 		private:
 			void usage(char * argv0);
 			void import_opml(const char * filename);
diff --git a/include/rss.h b/include/rss.h
index e4dfd4ea..f23d245b 100644
--- a/include/rss.h
+++ b/include/rss.h
@@ -43,6 +43,8 @@ namespace newsbeuter {
 				return pubDate_;
 			}
 			void set_pubDate(time_t t);
+
+			bool operator<(const rss_item& item) const { return item.pubDate_ < this->pubDate_; } // new items come first
 			
 			inline const std::string& guid() const { return guid_; }
 			void set_guid(const std::string& g);
@@ -50,6 +52,7 @@ namespace newsbeuter {
 			inline bool unread() const { return unread_; }
 			void set_unread(bool u);
 			void set_unread_nowrite(bool u);
+			void set_unread_nowrite_notify(bool u);
 			
 			inline void set_cache(cache * c) { ch = c; }
 			inline void set_feedurl(const std::string& f) { feedurl_ = f; }
@@ -110,7 +113,7 @@ namespace newsbeuter {
 			rss_item& get_item_by_guid(const std::string& guid);
 			
 			inline const std::string& rssurl() const { return rssurl_; }
-			inline void set_rssurl(const std::string& u) { rssurl_ = u; }
+			void set_rssurl(const std::string& u);
 			
 			unsigned int unread_item_count() const;
 
@@ -121,6 +124,10 @@ namespace newsbeuter {
 			virtual bool has_attribute(const std::string& attribname);
 			virtual std::string get_attribute(const std::string& attribname);
 
+			void update_items(std::vector<rss_feed>& feeds);
+
+			inline void set_query(const std::string& s) { query = s; }
+
 		private:
 			std::string title_;
 			std::string description_;
@@ -129,6 +136,7 @@ namespace newsbeuter {
 			std::string rssurl_;
 			std::vector<rss_item> items_;
 			std::vector<std::string> tags_;
+			std::string query;
 			
 			cache * ch;
 	};
diff --git a/src/cache.cpp b/src/cache.cpp
index 5cdcbc8d..f92b913c 100644
--- a/src/cache.cpp
+++ b/src/cache.cpp
@@ -235,6 +235,9 @@ void cache::populate_tables() {
 
 // this function writes an rss_feed including all rss_items to the database
 void cache::externalize_rssfeed(rss_feed& feed) {
+	if (feed.rssurl().substr(0,6) == "query:")
+		return;
+
 	mtx->lock();
 	std::ostringstream query;
 	query << "SELECT count(*) FROM rss_feed WHERE rssurl = '" << feed.rssurl() << "';";
diff --git a/src/controller.cpp b/src/controller.cpp
index 2bd516ec..9ae9aa4a 100644
--- a/src/controller.cpp
+++ b/src/controller.cpp
@@ -335,7 +335,7 @@ void controller::mark_all_read(unsigned int pos) {
 		rsscache->catchup_all(feed.rssurl());
 		if (feed.items().size() > 0) {
 			for (std::vector<rss_item>::iterator it=feed.items().begin();it!=feed.items().end();++it) {
-				it->set_unread_nowrite(false);
+				it->set_unread_nowrite_notify(false);
 			}
 		}
 	}
diff --git a/src/itemview_formaction.cpp b/src/itemview_formaction.cpp
index 43d58b59..38c712bc 100644
--- a/src/itemview_formaction.cpp
+++ b/src/itemview_formaction.cpp
@@ -30,15 +30,17 @@ void itemview_formaction::prepare() {
 		rss_item& item = feed->get_item_by_guid(guid);
 		std::string code = "{list";
 
+		rss_feed * feedptr = item.get_feedptr();
+
 		code.append("{listitem text:");
 		std::ostringstream feedtitle;
 		feedtitle << _("Feed: ");
-		if (feed->title().length() > 0) {
-			feedtitle << feed->title();
-		} else if (feed->link().length() > 0) {
-			feedtitle << feed->link();
-		} else if (feed->rssurl().length() > 0) {
-			feedtitle << feed->rssurl();
+		if (feedptr->title().length() > 0) {
+			feedtitle << feedptr->title();
+		} else if (feedptr->link().length() > 0) {
+			feedtitle << feedptr->link();
+		} else if (feedptr->rssurl().length() > 0) {
+			feedtitle << feedptr->rssurl();
 		}
 		code.append(stfl::quote(feedtitle.str().c_str()));
 		code.append("}");
diff --git a/src/rss.cpp b/src/rss.cpp
index dd090936..aae67f3d 100644
--- a/src/rss.cpp
+++ b/src/rss.cpp
@@ -24,6 +24,7 @@ rss_parser::~rss_parser() { }
 
 rss_feed rss_parser::parse() {
 	rss_feed feed(ch);
+	bool skip_parsing = false;
 
 	feed.set_rssurl(my_uri);
 
@@ -62,140 +63,148 @@ rss_feed rss_parser::parse() {
 		std::string result = utils::run_filter(filter, buf);
 		GetLogger().log(LOG_DEBUG, "rss_parser::parse: output of `%s' is: %s", filter.c_str(), result.c_str());
 		err = mrss_parse_buffer(const_cast<char *>(result.c_str()), result.length(), &mrss);
+	} else if (my_uri.substr(0,6) == "query:") {
+
+		skip_parsing = true;
+
 	} else {
 		char buf[1024];
 		snprintf(buf, sizeof(buf), _("Error: unsupported URL: %s"), my_uri.c_str());
 		throw std::string(buf);
 	}
 
-	if (err != MRSS_OK) {
-		if (err == MRSS_ERR_POSIX) {
-			GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed with POSIX error: error = %s",strerror(errno));
-		}
-		GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed: err = %s (%d)",mrss_strerror(err), err);
-		GetLogger().log(LOG_USERERROR, "RSS feed `%s' couldn't be parsed: %s (error %d)", my_uri.c_str(), mrss_strerror(err), err);
-		if (mrss) {
-			mrss_free(mrss);
+	if (!skip_parsing) {
+
+		if (err != MRSS_OK) {
+			if (err == MRSS_ERR_POSIX) {
+				GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed with POSIX error: error = %s",strerror(errno));
+			}
+			GetLogger().log(LOG_ERROR,"rss_parser::parse: mrss_parse_url_with_options failed: err = %s (%d)",mrss_strerror(err), err);
+			GetLogger().log(LOG_USERERROR, "RSS feed `%s' couldn't be parsed: %s (error %d)", my_uri.c_str(), mrss_strerror(err), err);
+			if (mrss) {
+				mrss_free(mrss);
+			}
+			throw std::string(mrss_strerror(err));
 		}
-		throw std::string(mrss_strerror(err));
-	}
 
-	const char * encoding = mrss->encoding ? mrss->encoding : "utf-8";
+		const char * encoding = mrss->encoding ? mrss->encoding : "utf-8";
 
-	if (mrss->title) {
-		feed.set_title(utils::convert_text(mrss->title, "utf-8", encoding));
-	}
-	
-	if (mrss->description) {
-		feed.set_description(utils::convert_text(mrss->description, "utf-8", encoding));
-	}
+		if (mrss->title) {
+			feed.set_title(utils::convert_text(mrss->title, "utf-8", encoding));
+		}
+		
+		if (mrss->description) {
+			feed.set_description(utils::convert_text(mrss->description, "utf-8", encoding));
+		}
 
-	if (mrss->link) feed.set_link(mrss->link);
-	if (mrss->pubDate) 
-		feed.set_pubDate(parse_date(mrss->pubDate));
-	else
-		feed.set_pubDate(::time(NULL));
+		if (mrss->link) feed.set_link(mrss->link);
+		if (mrss->pubDate) 
+			feed.set_pubDate(parse_date(mrss->pubDate));
+		else
+			feed.set_pubDate(::time(NULL));
 
-	GetLogger().log(LOG_DEBUG, "rss_parser::parse: feed title = `%s' link = `%s'", feed.title().c_str(), feed.link().c_str());
+		GetLogger().log(LOG_DEBUG, "rss_parser::parse: feed title = `%s' link = `%s'", feed.title().c_str(), feed.link().c_str());
 
-	for (mrss_item_t * item = mrss->item; item != NULL; item = item->next ) {
-		rss_item x(ch);
-		if (item->title) {
-			std::string title = utils::convert_text(item->title, "utf-8", encoding);
-			x.set_title(title);
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted title `%s' to `%s'", item->title, title.c_str());
-		}
-		if (item->link) x.set_link(item->link);
-		if (item->author) {
-			x.set_author(utils::convert_text(item->author, "utf-8", encoding));
-		}
+		for (mrss_item_t * item = mrss->item; item != NULL; item = item->next ) {
+			rss_item x(ch);
+			if (item->title) {
+				std::string title = utils::convert_text(item->title, "utf-8", encoding);
+				x.set_title(title);
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted title `%s' to `%s'", item->title, title.c_str());
+			}
+			if (item->link) x.set_link(item->link);
+			if (item->author) {
+				x.set_author(utils::convert_text(item->author, "utf-8", encoding));
+			}
 
-		x.set_feedurl(feed.rssurl());
+			x.set_feedurl(feed.rssurl());
 
-		mrss_tag_t * content;
+			mrss_tag_t * content;
 
-		if (mrss_search_tag(item, "encoded", "http://purl.org/rss/1.0/modules/content/", &content) == MRSS_OK && content) {
-			/* RSS 2.0 content:encoded */
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: found content:encoded: %s\n", content->value);
-			if (content->value) {
-				std::string desc = utils::convert_text(content->value, "utf-8", encoding);
-				GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted description `%s' to `%s'", content->value, desc.c_str());
-				x.set_description(desc);
+			if (mrss_search_tag(item, "encoded", "http://purl.org/rss/1.0/modules/content/", &content) == MRSS_OK && content) {
+				/* RSS 2.0 content:encoded */
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: found content:encoded: %s\n", content->value);
+				if (content->value) {
+					std::string desc = utils::convert_text(content->value, "utf-8", encoding);
+					GetLogger().log(LOG_DEBUG, "rss_parser::parse: converted description `%s' to `%s'", content->value, desc.c_str());
+					x.set_description(desc);
+				}
+			} else {
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: found no content:encoded");
 			}
-		} else {
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: found no content:encoded");
-		}
 
-		if ((mrss->version == MRSS_VERSION_ATOM_0_3 || mrss->version == MRSS_VERSION_ATOM_1_0)) {
-			int rc;
-			if (((rc = mrss_search_tag(item, "content", "http://www.w3.org/2005/Atom", &content)) == MRSS_OK && content) ||
-			    ((rc = mrss_search_tag(item, "content", "http://purl.org/atom/ns#", &content)) == MRSS_OK && content)) {
-				GetLogger().log(LOG_DEBUG, "rss_parser::parse: found atom content: %s\n", content ? content->value : "(content = null)");
-				if (content && content->value) {
-					x.set_description(utils::convert_text(content->value, "utf-8", encoding));
+			if ((mrss->version == MRSS_VERSION_ATOM_0_3 || mrss->version == MRSS_VERSION_ATOM_1_0)) {
+				int rc;
+				if (((rc = mrss_search_tag(item, "content", "http://www.w3.org/2005/Atom", &content)) == MRSS_OK && content) ||
+					((rc = mrss_search_tag(item, "content", "http://purl.org/atom/ns#", &content)) == MRSS_OK && content)) {
+					GetLogger().log(LOG_DEBUG, "rss_parser::parse: found atom content: %s\n", content ? content->value : "(content = null)");
+					if (content && content->value) {
+						x.set_description(utils::convert_text(content->value, "utf-8", encoding));
+					}
+				} else {
+					GetLogger().log(LOG_DEBUG, "rss_parser::parse: mrss_search_tag(content) failed with rc = %d content = %p", rc, content);
 				}
 			} else {
-				GetLogger().log(LOG_DEBUG, "rss_parser::parse: mrss_search_tag(content) failed with rc = %d content = %p", rc, content);
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: not an atom feed");
 			}
-		} else {
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: not an atom feed");
-		}
 
-		/* last resort: search for itunes:summary tag (may be a podcast) */
-		if (x.description().length() == 0 && mrss_search_tag(item, "summary", "http://www.itunes.com/dtds/podcast-1.0.dtd", &content) == MRSS_OK && content) {
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: found itunes:summary: %s\n", content->value);
-			if (content->value) {
-				std::string desc = "<ituneshack>";
-				desc.append(utils::convert_text(content->value, "utf-8", encoding));
-				desc.append("</ituneshack>");
-				x.set_description(desc);
+			/* last resort: search for itunes:summary tag (may be a podcast) */
+			if (x.description().length() == 0 && mrss_search_tag(item, "summary", "http://www.itunes.com/dtds/podcast-1.0.dtd", &content) == MRSS_OK && content) {
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: found itunes:summary: %s\n", content->value);
+				if (content->value) {
+					std::string desc = "<ituneshack>";
+					desc.append(utils::convert_text(content->value, "utf-8", encoding));
+					desc.append("</ituneshack>");
+					x.set_description(desc);
+				}
+				
+			} else {
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: no luck with itunes:summary");
 			}
-			
-		} else {
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: no luck with itunes:summary");
-		}
 
-		if (x.description().length() == 0 && item->description) {
-			x.set_description(utils::convert_text(item->description, "utf-8", encoding));
-		}
+			if (x.description().length() == 0 && item->description) {
+				x.set_description(utils::convert_text(item->description, "utf-8", encoding));
+			}
 
-		if (item->pubDate) 
-			x.set_pubDate(parse_date(item->pubDate));
-		else
-			x.set_pubDate(::time(NULL));
-			
-		if (item->guid)
-			x.set_guid(item->guid);
-		else if (item->link)
-			x.set_guid(item->link); // XXX hash something to get a better alternative GUID
-		else if (item->title)
-			x.set_guid(item->title);
-		// ...else?! that's too bad.
-
-		if (item->enclosure_url) {
-			x.set_enclosure_url(item->enclosure_url);
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_url: %s", item->enclosure_url);
-		}
-		if (item->enclosure_type) {
-			x.set_enclosure_type(item->enclosure_type);
-			GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_type: %s", item->enclosure_type);
-		}
+			if (item->pubDate) 
+				x.set_pubDate(parse_date(item->pubDate));
+			else
+				x.set_pubDate(::time(NULL));
+				
+			if (item->guid)
+				x.set_guid(item->guid);
+			else if (item->link)
+				x.set_guid(item->link); // XXX hash something to get a better alternative GUID
+			else if (item->title)
+				x.set_guid(item->title);
+			// ...else?! that's too bad.
+
+			if (item->enclosure_url) {
+				x.set_enclosure_url(item->enclosure_url);
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_url: %s", item->enclosure_url);
+			}
+			if (item->enclosure_type) {
+				x.set_enclosure_type(item->enclosure_type);
+				GetLogger().log(LOG_DEBUG, "rss_parser::parse: found enclosure_type: %s", item->enclosure_type);
+			}
 
-		x.set_feedptr(&feed);
+			x.set_feedptr(&feed);
 
-		GetLogger().log(LOG_DEBUG, "rss_parser::parse: item title = `%s' link = `%s' pubDate = `%s' (%d) description = `%s'", 
-			x.title().c_str(), x.link().c_str(), x.pubDate().c_str(), x.pubDate_timestamp(), x.description().c_str());
+			GetLogger().log(LOG_DEBUG, "rss_parser::parse: item title = `%s' link = `%s' pubDate = `%s' (%d) description = `%s'", 
+				x.title().c_str(), x.link().c_str(), x.pubDate().c_str(), x.pubDate_timestamp(), x.description().c_str());
 
-		// only add item to feed if it isn't on the ignore list or if there is no ignore list
-		if (!ign || !ign->matches(&x)) {
-			feed.items().push_back(x);
-			GetLogger().log(LOG_INFO, "rss_parser::parse: added article title = `%s' link = `%s' ign = %p", x.title().c_str(), x.link().c_str(), ign);
-		} else {
-			GetLogger().log(LOG_INFO, "rss_parser::parse: ignored article title = `%s' link = `%s'", x.title().c_str(), x.link().c_str());
+			// only add item to feed if it isn't on the ignore list or if there is no ignore list
+			if (!ign || !ign->matches(&x)) {
+				feed.items().push_back(x);
+				GetLogger().log(LOG_INFO, "rss_parser::parse: added article title = `%s' link = `%s' ign = %p", x.title().c_str(), x.link().c_str(), ign);
+			} else {
+				GetLogger().log(LOG_INFO, "rss_parser::parse: ignored article title = `%s' link = `%s'", x.title().c_str(), x.link().c_str());
+			}
 		}
-	}
 
-	mrss_free(mrss);
+		mrss_free(mrss);
+
+	}
 
 	return feed;
 }
@@ -232,10 +241,18 @@ void rss_item::set_unread_nowrite(bool u) {
 	unread_ = u;
 }
 
+void rss_item::set_unread_nowrite_notify(bool u) {
+	unread_ = u;
+	if (feedptr)
+		feedptr->get_item_by_guid(guid_).set_unread_nowrite(unread_); // notify parent feed
+}
+
 void rss_item::set_unread(bool u) { 
 	if (unread_ != u) {
 		bool old_u = unread_;
 		unread_ = u;
+		if (feedptr)
+			feedptr->get_item_by_guid(guid_).set_unread_nowrite(unread_); // notify parent feed
 		try {
 			if (ch) ch->update_rssitem_unread_and_enqueued(*this, feedurl_); 
 		} catch (const dbexception& e) {
@@ -390,7 +407,8 @@ std::string rss_feed::description() const {
 rss_item& rss_feed::get_item_by_guid(const std::string& guid) {
 	for (std::vector<rss_item>::iterator it=items_.begin();it!=items_.end();++it) {
 		if (it->guid() == guid) {
-			it->set_feedptr(this);
+			if (rssurl_.substr(0,6) != "query:")
+				it->set_feedptr(this);
 			return *it;
 		}
 	}
@@ -519,3 +537,40 @@ bool rss_ignores::matches(rss_item* item) {
 	}
 	return false;
 }
+
+void rss_feed::update_items(std::vector<rss_feed>& feeds) {
+	if (query.length() == 0)
+		return;
+
+	GetLogger().log(LOG_DEBUG, "rss_feed::update_items: query = `%s'", query.c_str());
+
+	matcher m(query);
+
+	if (items_.size() > 0) {
+		items_.erase(items_.begin(), items_.end());
+	}
+
+	for (std::vector<rss_feed>::iterator it=feeds.begin();it!=feeds.end();++it) {
+		if (it->rssurl().substr(0,6) != "query:") { // don't fetch items from other query feeds!
+			for (std::vector<rss_item>::iterator jt=it->items().begin();jt!=it->items().end();++jt) {
+				if (m.matches(&(*jt))) {
+					GetLogger().log(LOG_DEBUG, "rss_feed::update_items: matcher matches!");
+					jt->set_feedptr(&(*it));
+					items_.push_back(*jt);
+				}
+			}
+		}
+	}
+
+	sort(items_.begin(), items_.end());
+}
+
+void rss_feed::set_rssurl(const std::string& u) {
+	rssurl_ = u;
+	if (rssurl_.substr(0,6) == "query:") {
+		std::vector<std::string> tokens = utils::tokenize_quoted(u, ":");
+		GetLogger().log(LOG_DEBUG, "rss_feed::set_rssurl: query name = `%s' expr = `%s'", tokens[1].c_str(), tokens[2].c_str());
+		set_title(tokens[1]);
+		set_query(tokens[2]);
+	}
+}
diff --git a/src/view.cpp b/src/view.cpp
index 97cb3ad1..ff3902e2 100644
--- a/src/view.cpp
+++ b/src/view.cpp
@@ -285,6 +285,7 @@ void view::set_tags(const std::vector<std::string>& t) {
 void view::push_itemlist(unsigned int pos) {
 	rss_feed * feed = ctrl->get_feed(pos);
 	GetLogger().log(LOG_DEBUG, "view::push_itemlist: retrieved feed at position %d (address = %p)", pos, feed);
+	feed->update_items(ctrl->get_all_feeds());
 	if (feed->items().size() > 0) {
 		itemlist->set_feed(feed);
 		itemlist->set_pos(pos);