Merge pull request #24 from xsco/enh/ep_schema_up_to_1_18_0

Support Engine Prime schema versions up to 1.18.0

69 files changed, 7854 insertions, 1607 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 73d9df2..e385f31 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,7 +6,7 @@
 # The meson/ninja build should be preferred in all other cases.
 #
 cmake_minimum_required(VERSION 3.10)
-project(libdjinterop VERSION 0.12.0)
+project(libdjinterop VERSION 0.13.0)
 
 # Require C++17
 set(CMAKE_CXX_STANDARD 17)
@@ -35,9 +35,17 @@ add_library(
     src/djinterop/enginelibrary/el_transaction_guard_impl.cpp
     src/djinterop/enginelibrary/encode_decode_utils.cpp
     src/djinterop/enginelibrary/performance_data_format.cpp
-    src/djinterop/enginelibrary/schema_1_6_0.cpp
-    src/djinterop/enginelibrary/schema_1_7_1.cpp
-    src/djinterop/enginelibrary/schema.cpp
+    src/djinterop/enginelibrary/schema/schema_1_6_0.cpp
+    src/djinterop/enginelibrary/schema/schema_1_7_1.cpp
+    src/djinterop/enginelibrary/schema/schema_1_9_1.cpp
+    src/djinterop/enginelibrary/schema/schema_1_11_1.cpp
+    src/djinterop/enginelibrary/schema/schema_1_13_0.cpp
+    src/djinterop/enginelibrary/schema/schema_1_13_1.cpp
+    src/djinterop/enginelibrary/schema/schema_1_13_2.cpp
+    src/djinterop/enginelibrary/schema/schema_1_15_0.cpp
+    src/djinterop/enginelibrary/schema/schema_1_17_0.cpp
+    src/djinterop/enginelibrary/schema/schema_1_18_0.cpp
+    src/djinterop/enginelibrary/schema/schema.cpp
     src/djinterop/crate.cpp
     src/djinterop/database.cpp
     src/djinterop/enginelibrary.cpp
@@ -47,7 +55,7 @@ add_library(
     src/djinterop/impl/database_impl.cpp
     src/djinterop/impl/track_impl.cpp
     src/djinterop/impl/transaction_guard_impl.cpp
-    src/djinterop/impl/util.cpp)
+        src/djinterop/util.cpp)
 
 target_compile_definitions(
     djinterop PUBLIC
diff --git a/example/engine_prime.cpp b/example/engine_prime.cpp
index ac81bef..3dcb957 100644
--- a/example/engine_prime.cpp
+++ b/example/engine_prime.cpp
@@ -33,6 +33,7 @@ int main(int argc, char** argv)
     auto db = el::create_or_load_database(dir, el::version_latest, created);
     std::cout << (created ? "Created " : "Loaded ") << "database in directory "
               << dir << std::endl;
+    std::cout << "DB version is " << db.version_name() << std::endl;
 
     for (auto& cr : db.crates())
     {
diff --git a/include/djinterop/database.hpp b/include/djinterop/database.hpp
index b947733..8be42ba 100644
--- a/include/djinterop/database.hpp
+++ b/include/djinterop/database.hpp
@@ -108,6 +108,9 @@ public:
     /// Returns the schema version of the database
     semantic_version version() const;
 
+    /// Returns a descriptive name for the database version.
+    std::string version_name() const;
+
     /// Removes a crate from the database
     ///
     /// All handles to that crate become invalid.
diff --git a/include/djinterop/enginelibrary.hpp b/include/djinterop/enginelibrary.hpp
index 588d293..9871d3c 100644
--- a/include/djinterop/enginelibrary.hpp
+++ b/include/djinterop/enginelibrary.hpp
@@ -29,6 +29,7 @@
 #include <vector>
 
 #include <djinterop/config.hpp>
+#include <djinterop/database.hpp>
 #include <djinterop/pad_color.hpp>
 #include <djinterop/semantic_version.hpp>
 
@@ -39,9 +40,48 @@ struct beatgrid_marker;
 
 namespace enginelibrary
 {
-static constexpr semantic_version version_1_6_0{1, 6, 0};
-static constexpr semantic_version version_1_7_1{1, 7, 1};
-static constexpr semantic_version version_latest = version_1_7_1;
+/// The schema associated with SC5000 Firmware 1.0.0.
+constexpr const semantic_version version_1_6_0{1, 6, 0};
+
+/// The schema associated with SC5000 Firmware 1.0.3.
+constexpr const semantic_version version_1_7_1{1, 7, 1};
+
+/// The schema associated with Engine Prime 1.1.1.
+constexpr const semantic_version version_1_9_1{1, 9, 1};
+
+/// The schema associated with SC5000 Firmware 1.2.0.
+constexpr const semantic_version version_1_11_1{1, 11, 1};
+
+/// The schema associated with SC5000 Firmware 1.2.2.
+constexpr const semantic_version version_1_13_0{1, 13, 0};
+
+/// The schema associated with Engine Prime 1.2.2.
+constexpr const semantic_version version_1_13_1{1, 13, 1};
+
+/// The schema associated with SC5000 Firmware 1.3.1.
+constexpr const semantic_version version_1_13_2{1, 13, 2};
+
+/// The schema associated with SC5000 Firmware 1.4.0.
+constexpr const semantic_version version_1_15_0{1, 15, 0};
+
+/// The schema associated with SC5000 Firmware 1.5.1/1.5.2.
+constexpr const semantic_version version_1_17_0{1, 17, 0};
+
+/// The schema associated with Engine Prime 1.5.1.
+constexpr const semantic_version version_1_18_0{1, 18, 0};
+
+/// Set of available schemas.
+constexpr const std::array<semantic_version, 10> all_versions{
+    version_1_6_0,  version_1_7_1,  version_1_9_1,  version_1_11_1,
+    version_1_13_0, version_1_13_1, version_1_13_2, version_1_15_0,
+    version_1_17_0, version_1_18_0,
+};
+
+/// The most recent schema version supported by the library.
+constexpr semantic_version version_latest = version_1_18_0;
+
+/// The most recent "firmware-usable" schema version supported by the library.
+constexpr semantic_version version_latest_firmware = version_1_17_0;
 
 namespace standard_pad_colors
 {
@@ -61,6 +101,9 @@ constexpr const std::array<pad_color, 8> pads{pad_1, pad_2, pad_3, pad_4,
 
 constexpr const char* default_database_dir_name = "Engine Library";
 
+/// Gets a descriptive name for a given schema version.
+std::string DJINTEROP_PUBLIC version_name(const semantic_version& version);
+
 /// Creates a new, empty database in a directory using the schema version
 /// provided.
 ///
@@ -68,7 +111,7 @@ constexpr const char* default_database_dir_name = "Engine Library";
 /// a database already exists in the target directory, an exception will be
 /// thrown.
 database DJINTEROP_PUBLIC create_database(
-    std::string directory,
+    const std::string& directory,
     const semantic_version& schema_version = version_latest);
 
 /// Create or load an Engine Library database in a given directory.
@@ -78,7 +121,7 @@ database DJINTEROP_PUBLIC create_database(
 /// database is returned.  The boolean reference parameter `created` can be used
 /// to determine whether the database was created or merely loaded.
 database DJINTEROP_PUBLIC create_or_load_database(
-    std::string directory, const semantic_version& schema_version,
+    const std::string& directory, const semantic_version& schema_version,
     bool& created);
 
 /// Returns a boolean indicating whether an Engine Library already exists in a
@@ -86,7 +129,7 @@ database DJINTEROP_PUBLIC create_or_load_database(
 bool DJINTEROP_PUBLIC database_exists(const std::string& directory);
 
 /// Loads an Engine Library database from a given directory.
-database DJINTEROP_PUBLIC load_database(std::string directory);
+database DJINTEROP_PUBLIC load_database(const std::string& directory);
 
 /// Given an Engine Library database, returns the path to its m.db sqlite
 /// database file
diff --git a/include/djinterop/track.hpp b/include/djinterop/track.hpp
index ff649cc..16d0f90 100644
--- a/include/djinterop/track.hpp
+++ b/include/djinterop/track.hpp
@@ -165,6 +165,8 @@ public:
     /// Returns the duration (metadata) of the track
     stdx::optional<std::chrono::milliseconds> duration() const;
 
+    // TODO (mr-smidge): Add `file_bytes()` and `set_file_bytes()` methods.
+
     /// Returns the file extension part of `track::relative_path()`
     ///
     /// An empty string is returned if the file doesn't have an extension.
@@ -304,6 +306,8 @@ public:
     void set_track_number(stdx::optional<int32_t> track_number) const;
     void set_track_number(int32_t track_number) const;
 
+    // TODO (mr-smidge): Add `uri()` and `set_uri()` methods.
+
     std::vector<waveform_entry> waveform() const;
 
     void set_waveform(std::vector<waveform_entry> waveform) const;
diff --git a/meson.build b/meson.build
index 575e12f..f2fb038 100644
--- a/meson.build
+++ b/meson.build
@@ -1,7 +1,7 @@
 project(
     'djinterop',
     'cpp', 'c',
-    version: '0.12.0',
+    version: '0.13.0',
     license: 'LGPL-3.0',
     default_options: ['cpp_std=c++17', 'default_library=both'])
 
diff --git a/src/djinterop/crate.cpp b/src/djinterop/crate.cpp
index 4f3fdaf..856ff28 100644
--- a/src/djinterop/crate.cpp
+++ b/src/djinterop/crate.cpp
@@ -20,7 +20,7 @@
 
 #include <djinterop/djinterop.hpp>
 #include <djinterop/impl/crate_impl.hpp>
-#include <djinterop/impl/util.hpp>
+#include <djinterop/util.hpp>
 
 namespace djinterop
 {
diff --git a/src/djinterop/database.cpp b/src/djinterop/database.cpp
index fcff0d3..5be4c78 100644
--- a/src/djinterop/database.cpp
+++ b/src/djinterop/database.cpp
@@ -19,10 +19,10 @@
 
 #include <djinterop/djinterop.hpp>
 #include <djinterop/enginelibrary/el_database_impl.hpp>
-#include <djinterop/enginelibrary/schema.hpp>
+#include <djinterop/enginelibrary/schema/schema.hpp>
 #include <djinterop/impl/database_impl.hpp>
-#include <djinterop/impl/util.hpp>
 #include <djinterop/transaction_guard.hpp>
+#include <djinterop/util.hpp>
 
 namespace djinterop
 {
@@ -124,6 +124,11 @@ semantic_version database::version() const
     return pimpl_->version();
 }
 
+std::string database::version_name() const
+{
+    return pimpl_->version_name();
+}
+
 database::database(std::shared_ptr<database_impl> pimpl) :
     pimpl_{std::move(pimpl)}
 {
diff --git a/src/djinterop/enginelibrary.cpp b/src/djinterop/enginelibrary.cpp
index c1a3f1a..562b0f6 100644
--- a/src/djinterop/enginelibrary.cpp
+++ b/src/djinterop/enginelibrary.cpp
@@ -15,78 +15,42 @@
     along with libdjinterop.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/stat.h>
 #include <cmath>
 #include <string>
-#if defined(_WIN32)
-#include <direct.h>
-#endif
 
 #include <djinterop/djinterop.hpp>
-#include <djinterop/enginelibrary/el_database_impl.hpp>
-#include <djinterop/enginelibrary/el_transaction_guard_impl.hpp>
+#include "enginelibrary/el_database_impl.hpp"
+#include "enginelibrary/el_transaction_guard_impl.hpp"
+#include "enginelibrary/schema/schema.hpp"
+#include "util.hpp"
 
-namespace djinterop
+namespace djinterop::enginelibrary
 {
-namespace enginelibrary
+/// Gets a descriptive name for a given schema version.
+std::string version_name(const semantic_version& version)
 {
-
-static bool dir_exists(const std::string& directory)
-{
-    struct stat buf;
-    return stat(directory.c_str(), &buf) == 0;
-}
-
-static void ensure_dir_exists(const std::string &directory, bool &created)
-{
-    created = false;
-    if (!dir_exists(directory))
-    {
-#if defined(_WIN32)
-        if (_mkdir(directory.c_str()) != 0)
-#else
-        if (mkdir(directory.c_str(), 0755) != 0)
-#endif
-        {
-            throw std::runtime_error{
-                "Failed to create directory"};
-        }
-
-        created = true;
-    }
+    auto schema_creator_validator =
+        schema::make_schema_creator_validator(version);
+    return schema_creator_validator->name();
 }
 
 database create_database(
-        std::string directory, const semantic_version& schema_version)
+    const std::string& directory, const semantic_version& schema_version)
 {
-    if (!el_storage::schema_version_supported(schema_version))
-    {
-        throw unsupported_database_version{"Unsupported database version",
-                                           schema_version};
-    }
-
-    // Ensure the target directory exists.
-    bool dir_created;
-    ensure_dir_exists(directory, dir_created);
-
-    // Create schema.
-    auto storage = std::make_shared<el_storage>(std::move(directory));
-    el_transaction_guard_impl trans{storage};
-    storage->create_and_validate_schema(schema_version);
-    database db{std::make_shared<el_database_impl>(storage)};
-    trans.commit();
-    return db;
+    auto storage = std::make_shared<el_storage>(directory, schema_version);
+    return database{std::make_shared<el_database_impl>(storage)};
 }
 
 database create_or_load_database(
-    std::string directory, const semantic_version& schema_version, bool &created)
+    const std::string& directory, const semantic_version& schema_version,
+    bool& created)
 {
-    if (database_exists(directory))
+    try
     {
         created = false;
         return load_database(directory);
     }
-    else
+    catch (database_not_found& e)
     {
         created = true;
         return create_database(directory, schema_version);
@@ -95,24 +59,22 @@ database create_or_load_database(
 
 bool database_exists(const std::string& directory)
 {
-    // We have to find out whether the engine library exists. Naively, we'd do
-    // this by checking if the m.db and p.db files exist. However, if a previous
-    // attempt to create the engine library failed after creating the files and
-    // before creating the schemata, then the files exist but the enginelibrary
-    // doesn't exist.
-    if (!dir_exists(directory))
+    try
+    {
+        load_database(directory);
+    }
+    catch (database_not_found& e)
     {
-        // No EL DB directory.
         return false;
     }
 
-    el_storage storage{std::move(directory)};
-    return storage.schema_created();
+    return true;
 }
 
-database load_database(std::string directory)
+database load_database(const std::string& directory)
 {
-    return database{std::make_shared<el_database_impl>(std::move(directory))};
+    auto storage = std::make_shared<el_storage>(directory);
+    return database{std::make_shared<el_database_impl>(storage)};
 }
 
 std::string music_db_path(const database& db)
@@ -185,5 +147,4 @@ std::string perfdata_db_path(const database& db)
     return db.directory() + "/p.db";
 }
 
-}  // namespace enginelibrary
-}  // namespace djinterop
+}  // namespace djinterop::enginelibrary
diff --git a/src/djinterop/enginelibrary/el_crate_impl.cpp b/src/djinterop/enginelibrary/el_crate_impl.cpp
index 0567b02..c67b1b8 100644
--- a/src/djinterop/enginelibrary/el_crate_impl.cpp
+++ b/src/djinterop/enginelibrary/el_crate_impl.cpp
@@ -145,10 +145,25 @@ crate el_crate_impl::create_sub_crate(std::string name)
             }
         };
 
-    storage_->db << "INSERT INTO Crate (title, path) VALUES (?, ?)"
-                 << name.data() << (path + name + ";");
-
-    int64_t sub_id = storage_->db.last_insert_rowid();
+    int64_t sub_id;
+    if (storage_->version >= version_1_9_1)
+    {
+        // Newer schemas consider crates to be a kind of 'list', and so the
+        // `Crate` table has been replaced with a VIEW onto `List`.  The main
+        // difference is that `List` does not have an integer primary key, so
+        // the new id will need to be determined in advance.
+        storage_->db << "SELECT IFNULL(MAX(id), 0) + 1 FROM Crate" >> sub_id;
+        storage_->db << "INSERT INTO Crate (id, title, path) VALUES (?, ?, ?)"
+                     << sub_id << name.data() << (path + name + ";");
+    }
+    else
+    {
+        // Older schema versions have a dedicated table for crates that has
+        // an integer primary key, which will be filled automatically.
+        storage_->db << "INSERT INTO Crate (title, path) VALUES (?, ?)"
+                     << name.data() << (path + name + ";");
+        sub_id = storage_->db.last_insert_rowid();
+    }
 
     storage_->db << "INSERT INTO CrateParentList (crateOriginId, "
                     "crateParentId) VALUES (?, ?)"
diff --git a/src/djinterop/enginelibrary/el_database_impl.cpp b/src/djinterop/enginelibrary/el_database_impl.cpp
index bb82e24..acd7fda 100644
--- a/src/djinterop/enginelibrary/el_database_impl.cpp
+++ b/src/djinterop/enginelibrary/el_database_impl.cpp
@@ -21,9 +21,9 @@
 #include <djinterop/enginelibrary/el_storage.hpp>
 #include <djinterop/enginelibrary/el_track_impl.hpp>
 #include <djinterop/enginelibrary/el_transaction_guard_impl.hpp>
-#include <djinterop/enginelibrary/schema.hpp>
-#include <djinterop/impl/util.hpp>
+#include <djinterop/enginelibrary/schema/schema.hpp>
 #include <djinterop/transaction_guard.hpp>
+#include <djinterop/util.hpp>
 
 namespace djinterop
 {
@@ -50,14 +50,6 @@ void ensure_valid_crate_name(const std::string& name)
 
 }  // namespace
 
-el_database_impl::el_database_impl(std::string directory) :
-    storage_{std::make_shared<el_storage>(std::move(directory))}
-