Skip to content
Snippets Groups Projects
Commit c96d6da8 authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

add caching and versioning parameters; add generated column datasize for document fragments

parent 23d9c4e0
No related branches found
No related tags found
1 merge request!1Draft: Migration
......@@ -12,6 +12,10 @@ SELECT * FROM dstorage.users
-- :doc retrieve a list of all repositories.
SELECT * FROM :i:repositories-table
-- :name get-repository-info :? :1
-- :doc retrieve repository infos.
SELECT * FROM :i:repositories-table WHERE id = :repository-name
-- :name get-document-frame :? :1
-- :doc retrieve a document-frame given the uuid.
......
......@@ -9,6 +9,7 @@ CREATE TABLE :i:repository-table (
reference varchar(32) NULL,
created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
"data" bytea NOT NULL,
datasize numeric NULL GENERATED ALWAYS AS (length("data")) STORED,
"version" int8 NOT NULL DEFAULT 0,
data_encoding :i:ds-encoding-fqn NOT NULL,
CONSTRAINT
......@@ -24,8 +25,8 @@ CREATE INDEX
ON :i:repository-table USING btree (document_id);
--;;
INSERT INTO :i:repositories-table
(id)
VALUES (:repository-name)
(id, caching, versioning)
VALUES (:repository-name, :caching, :versioning)
-- :name delete-repository!
......
......@@ -6,6 +6,8 @@ CREATE SCHEMA IF NOT EXISTS :i:schema-name;
--;;
CREATE TABLE IF NOT EXISTS :i:repositories-table (
id varchar(32) NOT NULL,
caching bool NOT NULL DEFAULT FALSE,
"versioning" bool NOT NULL DEFAULT TRUE,
created_at timestamptz(0) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT repositories_pk PRIMARY KEY (id));
--;;
......
......@@ -11,28 +11,35 @@
[document-storage.query :refer [query-fits? update-queries
update-queries-remove-document]]
[conman.core :as conman]
[clj-helper.string :refer [get-unique-id]]))
[clj-helper.string :refer [get-unique-id]]
[clj-helper.vector :as vhelper]))
(defn prepare-repository-name [repo-id]
(-> (if (keyword? repo-id)
(name repo-id)
(str repo-id))
(str/replace "-" "_")
))
(str/replace "-" "_")))
(defprotocol db-storage
"Base Protocol for document storages"
(get-schema [this] "Return the db-schema name")
(create-repository [this repository] "Create a new repository")
(delete-repository [this repository] "Delete a repository")
(cached-repository? [this repository] "Checks if the repository will use caching")
(run-query [this query-key params] "Run a Query on a DB-based Storage"))
(defrecord postgres-storage [storage-id params connection queries]
db-storage
(get-schema [this]
(.-storage-id this))
(create-repository [this repository]
(.run-query this :create-repository! {:repository repository}))
(create-repository [this {:keys [caching versioning] :or {caching false
versioning true}
:as repository}]
(log/info repository)
(.run-query this :create-repository!
{:repository (:name repository)
:caching caching
:versioning versioning}))
(delete-repository [this repository]
(.run-query this :delete-repository! {:repository repository}))
(run-query [this query-key params]
......@@ -57,18 +64,25 @@
:uuid (get-unique-id "ds")}]
(conman/query (:queries this) query-key
(merge table-names params))))
(cached-repository? [this repository]
(let [repo-info (run-query this :get-repository-info
{:repository repository})]
(:caching repo-info)))
protocol/storage
(init-storage [this]
(log/info "Init Postgres-Storage.")
(.run-query this :create-storage! {})
(let [repositories (set (map prepare-repository-name (get (.-params this) :repositories [])))
(let [repositories (map (fn [[id repo-params]]
(assoc repo-params :name (prepare-repository-name id)))
(get (.-params this) :repositories {}))
repository-names (set (map :name repositories))
existing-repos (set (map :id (.list-repositories this)))]
(log/info "Repositories: " repositories)
(log/info "Repositories: " repository-names)
(log/info "Existing Repositories: " existing-repos)
(doseq [r (difference repositories existing-repos)]
(doseq [r (difference repository-names existing-repos)]
(log/info "Create Repository: " r)
(.create-repository this r))))
(.create-repository this (vhelper/get-by repositories :name r)))))
(list-repositories [this]
(run-query this :get-repositories {}))
......@@ -94,8 +108,9 @@
{:repository repository}))))
(versioned-repository? [this repository]
;;;TODO check if repo is versioned
true)
(let [repo-info (run-query this :get-repository-info
{:repository repository})]
(:versioning repo-info)))
(find-documents [this query repository]
(let [docs (.list-documents this repository)]
......@@ -103,7 +118,6 @@
(let [doc (.load-document this fname repository)]
(when (query-fits? doc query)
[fname doc])))
docs))]
result)))
......@@ -120,6 +134,9 @@
(defn make-postgres-ds [config]
(let [conn (db/connect! config)
id (str "ds" (name (or (:id config) "main")))
storage (->postgres-storage id {:repositories (:repositories config)} (:connection conn) (:queries conn))]
storage (->postgres-storage id
{:repositories (:repositories config)}
(:connection conn)
(:queries conn))]
(.init-storage storage)
storage))
......@@ -16,6 +16,7 @@
:id frame-id}))
(defn get-last-document-frame [storage document-id repository]
;;;TODO check if this can be speed up by using index in db
(when-let [frame (.run-query storage :get-last-document-frame
{:repository repository
:document-id document-id})]
......@@ -92,11 +93,11 @@
(defn delete-document [storage document-id repository]
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/delete
:encoding :ds-encoding/edn
:version (inc (get-last-document-version storage document-id repository))
:reference nil
:data ""}))
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/delete
:encoding :ds-encoding/edn
:version (inc (get-last-document-version storage document-id repository))
:reference nil
:data (encoding/encode-document "")}))
......@@ -14,7 +14,7 @@
(reset! ds-atom
(dspostgres/make-postgres-ds
{:id :testing
:repositories [:testrepo]
:repositories {:testrepo {}}
:jdbc-url (env :database-url)}))
(f)
(.delete-repository @ds-atom :testrepo))
......@@ -30,3 +30,6 @@
(deftest find-document-test
(protocol-tests/test-find-documents ds-atom))
;;;TODO test if versioning/caching parameters are set correctly for new repositories
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment