Merge branch 'release/0.5.0'v0.5.0

36 files changed, 8464 insertions, 2553 deletions

diff --git a/.gitignore b/.gitignore
index 4934913..9dab6d3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,5 +17,10 @@ CMakeFiles
 build.ninja
 rules.ninja
 cmake_install.cmake
+.cmake/
+_deps/
+bin/
+install_manifest.txt
 
 login_credentials.hpp
+
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..44b9a71
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,18 @@
+image: gcc
+
+build:
+  stage: build
+  before_script:
+    - apt update && apt -y install cmake make
+  script:
+    - cmake .
+    - make
+  artifacts:
+    paths:
+      - bin/rttt 
+
+# run tests using the binary built before
+test:
+  stage: test
+  script:
+    - run-parts --regex catch_ bin/
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index b0a49c9..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "third-party/imtui"]
-	path = third-party/imtui
-	url = https://github.com/ggerganov/imtui
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e25a7cc..80bd2d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
 cmake_minimum_required (VERSION 3.16)
-project(hnrtui)
+project(rttt)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS "on")
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
@@ -8,13 +8,23 @@ include(cmake/GitVars.cmake)
 
 # dependencies
 
-add_subdirectory(third-party)
+#add_subdirectory(third-party)
 
 find_package(CURL REQUIRED)
 
 include(FetchContent)
-FetchContent_Declare(cpr GIT_REPOSITORY https://github.com/libcpr/cpr.git
-                         GIT_TAG 1.7.2)
+FetchContent_Declare(pugixml 
+  GIT_REPOSITORY https://github.com/zeux/pugixml GIT_TAG v1.11.4)
+FetchContent_MakeAvailable(pugixml)
+
+FetchContent_Declare(imtui 
+  GIT_REPOSITORY https://github.com/ggerganov/imtui 
+  GIT_TAG "aa55479de6ea4d3bc0f730aaf91e20c299b61742")
+FetchContent_MakeAvailable(imtui)
+
+FetchContent_Declare(cpr 
+  GIT_REPOSITORY https://github.com/libcpr/cpr.git
+  GIT_TAG 1.7.2)
 FetchContent_MakeAvailable(cpr)
 
 FetchContent_Declare(json
@@ -51,43 +61,46 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
 endif()
 
 # Program name and sources
-set (TARGET hnrtui)
-set (SOURCES src/main.cpp src/hn-state.cpp src/impl-ncurses.cpp)
+set (TARGET rttt)
+set (SOURCES src/main.cpp)
 
 # Setup executable
 add_executable(${TARGET} ${SOURCES})
 
 target_include_directories(${TARGET} PRIVATE
   .
-  ${CURL_INCLUDE_DIR}
 )
 
 target_link_libraries(${TARGET} PRIVATE
-  imtui-ncurses
   ${CURL_LIBRARIES}
   cpr::cpr
   nlohmann_json::nlohmann_json
   Threads::Threads
+  pugixml::pugixml
+  imtui-ncurses
 )
 
-install(TARGETS hnrtui RUNTIME DESTINATION bin)
+target_compile_options(${TARGET} PRIVATE -Wall -Wextra -Wpedantic -Werror)
+
+install(TARGETS ${TARGET} RUNTIME DESTINATION bin)
 
 FILE(GLOB TESTFILES test/catch_*.cpp)
 foreach(TESTFILE ${TESTFILES})
-    get_filename_component(NAME ${TESTFILE} NAME_WE) 
-    add_executable(${NAME} ${TESTFILE} src/hn-state.cpp src/impl-ncurses.cpp)
-    target_include_directories(${NAME} PUBLIC)
+  get_filename_component(NAME ${TESTFILE} NAME_WE) 
+  if (NOT "catch_login" STREQUAL ${NAME} OR EXISTS "test/login_credentials.hpp")
+    add_executable(${NAME} ${TESTFILE})
     target_include_directories(${NAME} PRIVATE
       .
-      ${CURL_INCLUDE_DIR}
       test/include/ 
       src/
     )
     target_link_libraries(${NAME} PRIVATE
-      imtui-ncurses
       ${CURL_LIBRARIES}
       cpr::cpr
       nlohmann_json::nlohmann_json
       Threads::Threads
+      pugixml::pugixml
+      imtui-ncurses
     )
+  endif()
 endforeach()
diff --git a/README.md b/README.md
index fc7585e..fed4f91 100644
--- a/README.md
+++ b/README.md
@@ -1,18 +1,22 @@
-## Details
+# Read the things tui
 
-HNRTui is a small console application written in C++ for browsing [Hacker News](https://news.ycombinator.com/news) and Reddit. It is based on [hnterm](https://github.com/ggerganov/hnterm.git), but with added support for Reddit and more functionality. 
+Read-the-things-tui (rttt) lets you read the things from the terminal. The things currently include RSS/Atom, hackernews and Reddit. It is a terminal application written in C++ and originally based on [hnterm](https://github.com/ggerganov/hnterm.git), but with added support for other things (Reddit/RSS/Atom). 
 
 ## Installing
 
-## Building from source
-
-### Linux and Mac:
-
 ```bash
-#git clone https://github.com/ggerganov/hnterm --recursive
-cd hnrtui
+git clone https://gitlab.com/BlackEdder/rttt.git 
+cd rttt 
 cmake . 
-make
-
-./bin/hnrtui
+make install
 ```
+
+## Usage
+
+Press (?) to open a popup with all the available key bindings. Scrolling is done with 'j' and 'k', opening comments to a story by pressing 'l' and 'h' will bring you back to the main listing. 'o' allows you to open the url in a browser. 
+
+To switch to different readable things press '/' and type the various shortcuts to different items. For example, '/hn/top', '/hn/ask', '/hn/show' and '/hn/new' the for different hackernews pages. '/r/front' for the reddit front page and '/r/\<subreddit\>' for different subreddits. To see your rss/atom feed use '/rss'. Note that for rss support you need save an 'opml' file with your feeds as '.config/rttt/subscriptions.opml'.
+
+![screenshot](https://gitlab.com/BlackEdder/rttt/-/wikis/uploads/a6f6435591d28be270c8f783032dba37/rttt.png)
+
+
diff --git a/TODO.md b/TODO.md
index c0b5d0d..dd9e55d 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,43 +1,69 @@
-- [x] Implement folding (collapse)
-- [x] Remove all the duplication for top, new, show etc
-- [x] Only load the shown top, new, show etc. by default using ncurses?
-- [x] Explore cpr library use for reddit in catch_reddit
-- [x] Don't rely on HN specific things when showing stories
-- [x] Switch to a path based view for comments Path("/hn/story/comments/id")
-    - Use it to see if we need to display comments
-    - Use comments path for comments window/view in hn
-- [x] Common (minimal) item view. Only differentiate between story and comment for now.
-  - Get hackernews to use it to see if it is useable for them
-- [x] Get access token and start updating for reddit
-  - Call updateRequests in the render loop
-- [x] Implement getItems for reddit
-- [x] /r/front be one of the default windows
-- [x] Implement getComments(Path) for hackernews
-    - Render comments based on the above
-    - Call this from getItems when we need comments
-- [x] Implement getComments for Reddit
-- [x] Make sure ids are stored with every story and comment.. This can then later be used in the collapsed lookup
-- [x] Make collapsed state work. Comments do have ids. We could store it by path? (in a std::map<std::string, std::unordered_set>>)
-- [x] Switch using /hn/top, /hn/new, /r/front, /r/neovim etc.
-- [x] Check the ini support build into imgui, can we start using that for our settings as well?
-- [x] Reddit caches, rely on path.name, not the full request url
-- [x] Make force refresh work for Reddit
-- [ ] move toRender to be HN only
-- [ ] Implement some kind of error window (log them to status?)
-- [ ] Can I remove the storyid stuff and rely on the internal state (at least for reddit?)
-- [ ] Don't keep the top comment at the top (at least not if it is to large; do things contain a short selftext/body that could be shown if we move of the top story)
-- [ ] Cleanup status window and lines
-    - Consider always having a lower "header", similar to neovim (can use ImGuiWindowFlags_NoTitleBar to turn of titlebars)
+This is a very rough list of future todo items.
+
+# Documentation urls
+
+[imgui](https://pixtur.github.io/mkdocs-for-imgui/site/api-imgui/ImGui--Dear-ImGui-end-user/#settingsini-utilities)
+
+# TODOs
+
+- [x] Build test without login_credentials (i.e. disable the test that needs login_credentials)
+- [x] Scrolling for large bits of text should be fixed
+- [x] Reimplement Tab to go through the pathCache
+- [x] RSS fixes
+    - [x] Story layout
+    - [x] Make sure id is unique (it currently isn't) (use the hash of (feed url + std::to_string(time)))
+    - [x] Get title from subscriptions.opml to use as key (maybe clean it up?)
+    - 'l' should open up the feed with full text, with the current item highlighted
+        - Since we don't know the feed name at the start (when we need keys) we'll have to store the feedurl in the Story, so we can use that to get all the stories
+        - Probably needs a "tinyurl" thingie for the feeds. (Use interning as well) (use std::hash, although note that this might be different from one run to another https://stackoverflow.com/questions/8029121/how-to-hash-stdstring)
+    - 'o' does not always seem to work
+- [x] Seriously think about release (0.5.0)
+    - [x] Reddit login? Maybe with an command line flag to do the first negotiation of tokens. State to send should be unique, so it is different for each device. Saved in config I guess
+    - [x] Run if subscriptions.opml I'd not present and mention RSS support on readme
+    - [x] Write README
+      - Add screenshots
+    - [x] Add hackernews reddit and rss/atom to the topics of the repository
+- [ ] Reddit login (so we can have better front page with our subreddits)
+- [ ] Reloading on new access token does not seem to work, probably because of the time out with mark_for_update? Maybe I should call on_timeout() directly? Or add a mark_for_update(force) option?
+- [ ] Make Path useable as map key (comparison based on name)
+    - Use it for the url keys in hackernews and reddit. 
+    - Easiest way is by implementing comparison operator: `auto operator<=>(Class1 const &other) const { return this->name <=> other.name;};`
+    - Might want to hash it?
+    - https://oleksandrkvl.github.io/2021/04/02/cpp-20-overview.html#three-way-comparison (should we imply strict ordering)?
+    - Seems like we don't need an equivalence operator (`==`) https://stackoverflow.com/questions/20168173/when-using-stdmap-should-i-overload-operator-for-the-key-type
+- [ ] Mark read state
+    - [ ] Could use J for mark read and go down
+- [ ] Explore performance
+    - [ ] Track how many items are "visible" (e.g. above the top of the window + actually visible items). And use that to get a reasonable number of items. This might be complicated for collapsed, but since we collapse them/never load them it might also just be fine. Something to keep in mind though! ;)
+      - Stop drawing couple items after they are not visible any more!
+    - [ ] Consider not calling active_storage.update() on every frame
+
+- [ ] Currently r/neovim and /r/neovim both work, think this is nice, but we should "convert" r/neovim to /r/neovim
+- [ ] Different containers where appropriate. Note that drawItems only expects iterators. Does not rely on indexable anywhere, so we can easily pass non vectors to it. I.e. in rss.hpp a set with custom sort order might be better.
+- [ ] Support reddit login (catch_login contains a (mostly worked out) example with refresh token etc; I think that here it would be nice to start using a state object for reddit, similar as for rss and hackernews)
+- [ ] Consistent naming style. Let's try the standard library style -> underscore and lowercase everywhere, use namespaces to differentiate: request::state state; etc.
+- [ ] If we are in non text mode and we press v, while on a large_item we end up with the highlighted item off screen. Presumably because scrolltop is wrong
+- [ ] Introduce View wrapper to the Story and Comment items, which gives consistent access to different fields
+    - [ ] This should probably also store collapsed?
+- [ ] Can we add feeds in a staggered manner (time wise)
+- [ ] Get rid of the UI namespace (incorporate it into the new ui)
+- [ ] Get rid of linearise either by loading kids for reddit similar to current hackernews implementation or writing a nice iterator (see the good youtube video I saved on newpipe)
+- [ ] Ways to cleanup different caches (especially story paths) Probably need a general way of storing things with a time of when it was last stored/updated. Then we can start using that instead of the various caches we use.
+- [ ] Performance analysis
+    - [ ] Ticker in active_storage, or just perform certain things less often (in main)
+- [ ] markForUpdate (or similar) on error?
+- [ ] Can we use decltype to simplify some of the variant code? Maybe with templated functions?
+- [ ] Increase default content size, so we can hold more info (double check this is needed, because I have conflicting experiences).
+- [ ] Explore other fonts/utf8. Would allow us to prettify things (nicer highlight marker as well) (first one has useful code snippets; https://pixtur.github.io/mkdocs-for-imgui/site/FONTS/) (https://pixtur.github.io/mkdocs-for-imgui/site/FAQ/#q-how-can-i-easily-use-icons-in-my-application) (https://pixtur.github.io/mkdocs-for-imgui/site/FAQ/#q-how-can-i-display-and-input-non-latin-characters-such-as-chinese-japanese-korean-cyrillic) 
+- [ ] Open images/youtube using run-mailcap like ttrv allowed me to do
+    It seemed to infer the type from the link (run-mailcap does not automatically do that) and then pass it to the correct thing based on .mailcap. For example when it gets a youtube link it knows it is mime type video/x-youtube and passes that along to the runner (run-mailcap?) together with the link?.. Can I force give the mime-type to run-mailcap?
+    - Easiest will be to use have specific options in a config file (i.reddit -> eog, www.youtube -> mpv (see my .mailcap)), and use xdg-open otherwise.
 - [ ] Reddit: limit the requested items to the amount needed (should be possible with the correct parameters)
     - [ ] Pass maxStories to the reddit updatePath?
     - [ ] Clean way to load additional stories.. There was something in the API for this
-- [ ] Implement a cache (e.g. for 10 paths) (should the cache be on the disk?)
-    - Note that when asking for a comments etc we should check this cache..
-    - Might need to be HN/Reddit specific
-- [ ] Replace impl_ncurses with a cpr based one for hpr as well
-    - I think we can easily just use the queuing already present in reddit namespace
-    - Remove libcurl dependency from CMakeLists.txt
-- [ ] Use Fetch_Content instead of git submodules (third-party)
+    - https://towardsdatascience.com/how-to-use-the-reddit-api-in-python-5e05ddfd1e5c has some information on getting more than a 100 using before and after
+- [ ] Use system libraries? And/or Fetch_Content instead of git submodules (third-party)
+    - Ideally support both methods
 - [ ] Support ctest
 - [ ] Support installation (are we linking to libraries or can we pull them all in)
 - [ ] AUR package
@@ -45,13 +71,11 @@
 - [ ] Login to hackernews?
 - [ ] Don't require login reddit?
     - https://fusionauth.io/learn/expert-advice/oauth/modern-guide-to-oauth/
-- [ ] Push it to gitlab
 - [ ] Voting
-- [ ] Open images/youtube using run-mailcap like ttrv allowed me to do
 - [ ] Posting of comments
 - [ ] Tab completion of subreddits and other paths when logged in
-- [ ] Refactor drawing to be more readable (draw(hnrtui::Item) -> (draw(hnrtui::Story), draw(hrntui::Comment) etc)
-- [ ] ImGui difference between begin and childbegin
-- [ ] Design more like ttrv (took a photo on my phone.. I especially like there selection marker
 - [ ] Fix mouse support?
-- [ ] Make sure selected story is still in the window when switching view mode
+    - [ ] Can we use begin/endcombo to make whole items clickable?
+- [ ] Maybe we should use the build in scrolling for the story list window
+- [ ] Fix help window (not sure when it broke???)
+- [ ] Use raw literals wherever possible? https://www.geeksforgeeks.org/raw-string-literal-c/
diff --git a/src/config.hpp b/src/config.hpp
new file mode 100644
index 0000000..0e07269
--- /dev/null
+++ b/src/config.hpp
@@ -0,0 +1,59 @@
+#pragma once
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+
+#include "nlohmann/json.hpp"
+
+#include "rttt.hpp"
+
+namespace rttt {
+namespace config {
+
+inline std::filesystem::path getPath() {
+  auto path_char = std::getenv("XDG_CONFIG_HOME");
+  if (path_char != NULL)
+    return std::filesystem::path(path_char);
+  std::filesystem::path path =
+      std::string(std::getenv("HOME")) + "/.config/rttt";
+  return std::filesystem::absolute(path);
+}
+
+inline std::filesystem::path getFilePath() { return getPath() / "config.json"; }
+
+inline nlohmann::json defaultConfig() {
+  nlohmann::json config;
+  config["reddit"]["client_id"] = "8yDBiibHONI95SeMWLZspg";
+  config["reddit"]["device_id"] = rttt::random_string(25);
+  return config;
+}
+
+inline bool save(const nlohmann::json &json) {
+  std::ofstream file;
+  file.open(getFilePath().c_str());
+  file << json.dump(2) << std::endl;
+  file.close();
+  return true;
+}
+
+inline nlohmann::json load() {
+  std::filesystem::path fn = "config.json";
+  auto path = getPath();
+  nlohmann::json config;
+  if (!std::filesystem::exists(path)) {
+    std::filesystem::create_directory(path);
+  }
+  auto fpath = getFilePath();
+  if (!std::filesystem::exists(fpath)) {
+    config = defaultConfig();
+    save(config);
+  } else {
+    FILE *pFile;
+    pFile = fopen(fpath.c_str(), "r");
+    config = nlohmann::json::parse(pFile);
+  }
+  return config;
+}
+} // namespace config
+} // namespace rttt
diff --git a/src/hackernews.hpp b/src/hackernews.hpp
index 384546a..01df813 100644
--- a/src/hackernews.hpp
+++ b/src/hackernews.hpp
@@ -1,81 +1,363 @@
 #pragma once
 
-#include "hn-state.h"
-#include "hnrtui.hpp"
+#include <map>
+#include <queue>
+#include <string>
+#include <unordered_set>
 
+#include "cpr/cpr.h"
+#include "nlohmann/json.hpp"
+
+#include "item.hpp"
+#include "logger.hpp"
+#include "request.hpp"
+#include "rttt.hpp"
+
+namespace rttt {
 namespace hackernews {
-static HN::ItemIds toUpdate;
-
-inline hnrtui::Item toItem(const HN::Item &hnitem) {
-  hnrtui::Item item;
-
-  if (std::holds_alternative<HN::Story>(hnitem.data)) {
-    auto hndata = std::get<HN::Story>(hnitem.data);
-    hnrtui::Story itemdata;
-    itemdata.title = hndata.title;
-    itemdata.domain = hndata.domain;
-    itemdata.time = hndata.time;
-    itemdata.score = hndata.score;
-    itemdata.descendants = hndata.descendants;
-    itemdata.url = hndata.url;
-    itemdata.by = hndata.by;
-    itemdata.text = hndata.text;
-    itemdata.id = std::to_string(hndata.id);
-    item.data = itemdata;
-  } else if (std::holds_alternative<HN::Comment>(hnitem.data)) {
-    hnrtui::Comment itemdata;
-    auto hndata = std::get<HN::Comment>(hnitem.data);
-    itemdata.by = hndata.by;
-    itemdata.text = hndata.text;
-    itemdata.time = hndata.time;
-    itemdata.id = std::to_string(hndata.id);
-    item.data = itemdata;
-  } else {
-    hnrtui::Story itemdata;
-    itemdata.title = "Unsupported type";
-    item.data = itemdata;
+
+using ItemData = std::map<std::string, std::string>;
+
+namespace {
+rttt::request::State<std::string> pathRequestCache;
+rttt::request::State<ItemId> itemRequestCache;
+} // namespace
+
+struct Job {
+  std::string by = "";
+  ItemId id = 0;
+  int score = 0;
+  uint64_t time = 0;
+  std::string title = "";
+  std::string url = "";
+  std::string domain = "";
+};
+
+struct Poll {
+  std::string by = "";
+  int descendants = 0;
+  ItemId id = 0;
+  ItemIds kids;
+  ItemIds parts;
+  int score = 0;
+  uint64_t time = 0;
+  std::string text = "";
+  std::string title = "";
+};
+
+struct PollOpt {
+  std::string by = "";
+  ItemId id = 0;
+  ItemId poll = 0;
+  int score = 0;
+  uint64_t time = 0;
+  std::string text = "";
+};
+
+using ItemContainer = rttt::active_storage<int, rttt::ItemVariant>;
+
+inline std::vector<std::pair<size_t, rttt::ItemVariant>>
+get_kids_with_indent(const rttt::ItemVariant &hnitem, size_t indent,
+                     ItemContainer &known_hnitems,
+                     rttt::active_storage<std::string, int> &collapsed,
+                     int head) {
+
+  std::vector<std::pair<size_t, rttt::ItemVariant>> items;
+  if (head <= 0)
+    return items;
+
+  hackernews::ItemIds kids;
+  if (std::holds_alternative<hackernews::Story>(hnitem))
+    kids = std::get<hackernews::Story>(hnitem).kids;
+  else if (std::holds_alternative<hackernews::Comment>(hnitem))
+    kids = std::get<hackernews::Comment>(hnitem).kids;
+  else if (std::holds_alternative<rttt::Unknown>(hnitem))
+    return items;
+  else
+    assert(false);
+
+  for (auto &id : kids) {
+    if (!known_hnitems.contains(id)) {
+      known_hnitems.insert({id, hackernews::Comment()});
+    }
+    known_hnitems.mark_active(id);
+
+    auto &kid_hnitem = known_hnitems.at(id);
+    if (!collapsed.contains(std::to_string(id))) {
+      // In case it has been toggled before
+      kid_hnitem = rttt::unset_collapsed(std::move(kid_hnitem));
+      items.push_back({indent, kid_hnitem});
+      auto recursive_kids = hackernews::get_kids_with_indent(
+          kid_hnitem, indent + 1, known_hnitems, collapsed,
+          head - items.size());
+      items.insert(items.end(), recursive_kids.begin(), recursive_kids.end());
+    } else {
+      kid_hnitem = rttt::set_collapsed(std::move(kid_hnitem));
+      items.push_back({indent, kid_hnitem});
+      collapsed.mark_active(std::to_string(id));
+    }
   }
+  return items;
+}
 
-  return item;
+template <typename T>
+T safe_parse(const nlohmann::json &json, std::string key) {
+  if (json.contains(key))
+    return json[key].get<T>();
+  return T();
 }
 
-inline hnrtui::Item addKids(hnrtui::Item &&item, const HN::Item &hnitem,
-                            const std::map<int, HN::Item> items,
-                            HN::ItemIds &toRefresh, bool recursively = true) {
-  if (std::holds_alternative<HN::Story>(hnitem.data)) {
-    assert(std::holds_alternative<hnrtui::Story>(item.data));
-    auto &itemdata = std::get<hnrtui::Story>(item.data);
-    auto hndata = std::get<HN::Story>(hnitem.data);
-    for (auto &id : hndata.kids) {
-      toRefresh.push_back(id);
-      if (!items.contains(id)) {
-        toUpdate.push_back(id);
+template <typename T> T parseData(const nlohmann::json &json) {
+  T data;
+
+  if constexpr (std::is_same<T, Comment>::value ||
+                std::is_same<T, Story>::value || std::is_same<T, Job>::value ||
+                std::is_same<T, Poll>::value ||
+                std::is_same<T, PollOpt>::value) {
+    data.by = safe_parse<std::string>(json, "by");
+    data.id = safe_parse<int>(json, "id");
+    data.time = safe_parse<size_t>(json, "time");
+  }
+  if constexpr (std::is_same<T, Comment>::value ||
+                std::is_same<T, Story>::value || std::is_same<T, Poll>::value) {
+    data.kids = safe_parse<std::vector<int>>(json, "kids");
+    data.text = rttt::parseHTML(safe_parse<std::string>(json, "text"));
+  }
+  if constexpr (std::is_same<T, Story>::value || std::is_same<T, Job>::value ||
+                std::is_same<T, Poll>::value) {
+    data.score = safe_parse<int>(json, "score");
+    data.title = rttt::parseHTML(safe_parse<std::string>(json, "title"));
+  }
+  if constexpr (std::is_same<T, Story>::value || std::is_same<T, Job>::value) {
+    data.url = safe_parse<std::string>(json, "url");
+    int slash = 0;
+    for (auto &ch : data.url) {
+      if (ch == '/') {
+        ++slash;
         continue;
       }
-      auto hnit = items.at(id);
-      auto it = toItem(hnit);
-      if (recursively)
-        it = addKids(std::move(it), hnit, items, toRefresh, recursively);
-      itemdata.kids.push_back(it);
+      if (slash > 2)
+        break;
+      if (slash > 1)
+        data.domain += ch;
     }
-  } else if (std::holds_alternative<HN::Comment>(hnitem.data)) {
-    assert(std::holds_alternative<hnrtui::Comment>(item.data));
-    auto &itemdata = std::get<hnrtui::Comment>(item.data);
-    auto hndata = std::get<HN::Comment>(hnitem.data);
-    for (auto &id : hndata.kids) {
-      toRefresh.push_back(id);
-      if (!items.contains(id)) {
-        toUpdate.push_back(id);
-        continue;
+  }
+  if constexpr (std::is_same<T, Comment>::value) {
+    data.parent = safe_parse<int>(json, "parent");
+  }
+  if constexpr (std::is_same<T, Story>::value || std::is_same<T, Poll>::value) {
+    data.descendants = safe_parse<int>(json, "descendants");
+  }
+  if constexpr (std::is_same<T, Poll>::value) {
+    data.kids = safe_parse<std::vector<int>>(json, "kids");
+  }
+  if constexpr (std::is_same<T, PollOpt>::value) {
+    data.score = safe_parse<int>(json, "score");
+    data.text = rttt::parseHTML(safe_parse<std::string>(json, "text"));
+    data.poll = safe_parse<int>(json, "poll");
+  }
+  return data;
+}
+
+using URI = std::string;
+// https://github.com/HackerNews/API
+
+namespace {
+const URI kAPIItem = "https://hacker-news.firebaseio.com/v0/item/";
+const URI kAPITopStories =
+    "https://hacker-news.firebaseio.com/v0/topstories.json";
+const URI kAPINewStories =
+    "https://hacker-news.firebaseio.com/v0/newstories.json";
+const URI kAPIAskStories =
+    "https://hacker-news.firebaseio.com/v0/askstories.json";
+const URI kAPIShowStories =
+    "https://hacker-news.firebaseio.com/v0/showstories.json";
+const URI kAPIJobStories =
+    "https://hacker-news.firebaseio.com/v0/jobstories.json";
+const URI kAPIUpdates = "https://hacker-news.firebaseio.com/v0/updates.json";
+} // namespace
+
+inline URI getItemURI(ItemId id) {
+  return kAPIItem + std::to_string(id) + ".json";
+}
+
+inline ItemIds getChangedItemsIds() {
+  ItemIds data;
+  auto string_opt =
+      rttt::request::try_retrieve(pathRequestCache, std::string("/hn/updates"));
+  if (string_opt.has_value()) {
+    nlohmann::json json = nlohmann::json::parse(string_opt.value());
+    data = json["items"].get<ItemIds>();
+  }
+  return data;
+}
+
+enum class ItemType : int {
+  Unknown,
+  Story,
+  Comment,
+  Job,
+  Poll,
+  PollOpt,
+};
+
+inline ItemType get_item_type(const nlohmann::json &json) {
+  if (!json.contains("type"))
+    return ItemType::Unknown;
+
+  const std::string &strType = json["type"];
+
+  if (strType == "story")
+    return ItemType::Story;
+  if (strType == "comment")
+    return ItemType::Comment;
+  if (strType == "job")
+    return ItemType::Job;
+  if (strType == "poll")
+    return ItemType::Poll;
+  if (strType == "pollopt")
+    return ItemType::PollOpt;
+
+  return ItemType::Unknown;
+}
+
+struct State {
+
+  inline rttt::Path updatePath(rttt::Path &&path) {
+    assert(path.type == rttt::SiteType::HN);
+    if (path.mode == rttt::list_mode::story) {
+      assert(uriMap.contains(path.name));
+      pathRequestCache = rttt::request::push(
+          std::move(pathRequestCache), this->uriMap[path.name], path.name);
+    }
+    return std::move(path);
+  }
+
+  /**
+   * Update the content
+   *
+   * @param toRefresh Current visible ids
+   */
+  bool update() {
+    /*
+    My understanding of how this works:
+
+    activeItems are the currently visible stories
+    Every 30 seconds we update the changed/updated list based on kAPIUpdates
+    (updates.json) If changed && part of toRefresh then needUpdate and
+    needRequest = true
+    */
+    bool updated = false;
+
+    // Every 30 seconds check for updated item ids
+    if (rttt::timeout(lastUpdatePoll_s, 30)) {
+      pathRequestCache = rttt::request::push(std::move(pathRequestCache),
+                                             hackernews::kAPIUpdates,
+                                             std::string("/hn/updates"));
+
+      lastUpdatePoll_s = rttt::current_time();
+    }
+
+    {
+      auto ids = hackernews::getChangedItemsIds();
+      for (auto id : ids) {
+        if (!items.contains(id))
+          continue;
+        items.mark_for_update(id);
       }
-      auto hnit = items.at(id);
-      auto it = toItem(hnit);
-      if (recursively)
-        it = addKids(std::move(it), hnit, items, toRefresh,
-                     recursively);
-      itemdata.kids.push_back(it);
     }
+
+    // Note that this has to happen after getChangedItemsIds, because that one
+    // will empty the "/hn/updates" which requires special attention
+    auto maybe_receive_path =
+        rttt::request::try_pop_and_retrieve(pathRequestCache);
+    while (maybe_receive_path.has_value()) {
+      auto &pair = maybe_receive_path.value();
+      if (pair.first == "/hn/updates")
+        assert(false);
+      ItemIds ids = nlohmann::json::parse(pair.second);
+      idsMap[pair.first] = std::move(ids);
+      updated = true;
+      maybe_receive_path =
+          rttt::request::try_pop_and_retrieve(pathRequestCache);
+    }
+
+    auto maybe_re