Skip to content
Snippets Groups Projects
Commit 27d4e1a5 authored by Bruno Burke's avatar Bruno Burke :hamburger:
Browse files

fix find-documents in git storage, add versioned-repository? method

parent b1a9fff3
No related branches found
No related tags found
1 merge request!1Draft: Migration
Pipeline #93686 passed
......@@ -111,7 +111,11 @@
:cache (atom {})))
(log/error "Unable to initialize Document Storage - no config!")))
(list-repositories [this]
@repositories)
@repositories
(mapv
(fn [[id data]]
(assoc data :id id))
@repositories))
(load-document [this id repository]
(let [cached-repository (cached-repository? repository)]
(or
......@@ -160,16 +164,16 @@
files))
(find-documents [this query repository]
(let [docs (.list-documents this :repository repository)
(let [docs (.list-documents this repository)
cached-repository (cached-repository? repository)]
(or (when cached-repository
(when-let [cached-query (get-in @cache [repository :queries query])]
(into {} (map (fn [fname]
(let [doc (.load-document this fname :repository repository)]
(let [doc (.load-document this fname repository)]
[fname doc]))
cached-query))))
(let [result (into {} (keep (fn [fname]
(let [doc (.load-document this fname :repository repository)]
(let [doc (.load-document this fname repository)]
(when (query-fits? doc query)
[fname doc])))
......@@ -181,6 +185,9 @@
(pmap (fn [[id repo]]
(push-upstreams id)) @repositories))
(versioned-repository? [this repository]
(not (no-commit-repository? repository)))
(list-versions [this id repository]
(let [repo (get-repo repository)]
(sort-by :date (versioning/list-commits repo id))))
......
......@@ -6,15 +6,19 @@
(defn list-commits [repo path]
(-> repo
(jp/git-log :paths path)
(->>
(map #(hash-map
:commit (:id %)
:date (get-in % [:committer :date])
:version (jp/get-blob repo (:id %) path)))
(sort-by :date)
reverse)))
(-> repo
(jp/git-log :paths path)
(->>
(map #(let [blob (jp/get-blob repo (:id %) path)]
(hash-map
:commit (:id %)
:date (get-in % [:committer :date])
:version (jp/get-blob repo (:id %) path)
:type (if blob
:update
:delete))))
(sort-by :date)
reverse)))
(defn load-revision [commit repo path]
(let [git-repo (.getRepository repo)
......
......@@ -3,6 +3,7 @@
[clojure.java.io :as io]
[clojure.edn :refer [read-string]]
[clojure.set :refer [difference]]
[clojure.string :as str]
[fipp.edn :refer [pprint] :rename {pprint fipp}]
[document-storage.protocol :as protocol]
[document-storage.postgres.db :as db]
......@@ -12,6 +13,13 @@
[conman.core :as conman]
[clj-helper.string :refer [get-unique-id]]))
(defn prepare-repository-name [repo-id]
(-> (if (keyword? repo-id)
(name repo-id)
(str repo-id))
(str/replace "-" "_")
))
(defprotocol db-storage
"Base Protocol for document storages"
(get-schema [this] "Return the db-schema name")
......@@ -30,9 +38,7 @@
(run-query [this query-key params]
(let [storage-schema (.get-schema this)
repository (:repository params)
repository-name (if (keyword? repository)
(name repository)
(str repository))
repository-name (prepare-repository-name repository)
table-names {:schema-name storage-schema
:repository-name repository-name
:ds-encoding-fqn (str storage-schema
......@@ -56,7 +62,7 @@
(init-storage [this]
(log/info "Init Postgres-Storage.")
(.run-query this :create-storage! {})
(let [repositories (set (map name (get (.-params this) :repositories [])))
(let [repositories (set (map prepare-repository-name (get (.-params this) :repositories [])))
existing-repos (set (map :id (.list-repositories this)))]
(log/info "Repositories: " repositories)
(log/info "Existing Repositories: " existing-repos)
......@@ -74,7 +80,7 @@
(count (documents/load-document this id repository)))
(save-document [this id document repository options]
(documents/save-document this id document repository))
(documents/save-document this id document repository options))
(save-document [this id document repository]
(.save-document this id document repository {}))
......@@ -87,10 +93,14 @@
(map :document_id (run-query this :get-all-document-ids
{:repository repository}))))
(versioned-repository? [this repository]
;;;TODO check if repo is versioned
true)
(find-documents [this query repository]
(let [docs (.list-documents this :repository repository)]
(let [docs (.list-documents this repository)]
(let [result (into {} (keep (fn [fname]
(let [doc (.load-document this fname :repository repository)]
(let [doc (.load-document this fname repository)]
(when (query-fits? doc query)
[fname doc])))
......
......@@ -27,7 +27,7 @@
(defn restore-document [storage frame-id repository]
(when-let [frame (get-document-frame-by-id storage frame-id repository)]
(case (:data_type frame)
(case (name (:data_type frame))
"key" (encoding/decode-document (:data frame))
"diff" (let [data (encoding/decode-document (:data frame))]
(escript/patch
......@@ -47,15 +47,16 @@
(when-let [frame (get-document-frame-by-version storage document-id version repository)]
(restore-document storage (:id frame) repository))))
(defn save-keyframe [storage document-id document repository]
(let [doc (encoding/encode-document document)]
(defn save-keyframe [storage document-id document last-version repository]
(let [doc (encoding/encode-document document)
last-version (or last-version 0)]
;;VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
(.run-query storage :create-document-frame!
{:repository repository
:document-id document-id
:type :ds-type/key
:encoding :ds-encoding/nippy
:version 0
:version (inc last-version)
:reference nil
:data doc})))
......@@ -63,7 +64,7 @@
(defn save-pframe [storage document-id doc last-version repository]
(let [last-doc (load-document storage document-id last-version repository)
last-frame (get-document-frame-by-version storage document-id last-version repository)
diff (escript/diff last-doc doc)
diff (escript/diff last-doc doc {:str-diff? false :algo :quick})
diff-doc (encoding/encode-document (escript/get-edits diff))]
(.run-query storage :create-document-frame!
{:repository repository
......@@ -74,10 +75,12 @@
:reference (:id last-frame) ;;;FIXME save ID in reference not version
:data diff-doc})))
(defn save-document [storage document-id document repository]
(defn save-document [storage document-id document repository options]
(if-let [last-version (get-last-document-version storage document-id repository)]
(save-pframe storage document-id document last-version repository)
(save-keyframe storage document-id document repository)))
(if (true? (:force-keyframe options))
(save-keyframe storage document-id document last-version repository)
(save-pframe storage document-id document last-version repository))
(save-keyframe storage document-id document -1 repository)))
(defn delete-document [storage document-id repository]
......
......@@ -10,6 +10,7 @@
(list-repositories [ds] "List all repositories")
(list-versions [ds id repository] "List all version of a specific document in a repository. The result is expected to be sorted in ascending order.")
(load-version [ds id version repository] "Load a specific document version in a repository")
(versioned-repository? [ds repository] "Check if repository is versioned")
(get-document-size [ds id repository] "Calculate the current document size")
(init-storage [ds] "Start a document storage")
(shutdown [ds] "Stop a document storage"))
......@@ -44,6 +44,8 @@
(deftest versioning-test
(protocol-tests/test-versioning ds-atom))
(deftest find-document-test
(protocol-tests/test-find-documents ds-atom))
(deftest views-test
(testing "save views, save basic structure, load it and load views"
......
......@@ -27,3 +27,6 @@
(deftest versioning-test
(protocol-tests/test-versioning ds-atom))
(deftest find-document-test
(protocol-tests/test-find-documents ds-atom))
......@@ -46,3 +46,22 @@
(is (= (diff structure-v2 structure-v3)
(diff storage-doc-v2 storage-doc-v3))))))))
(defn test-find-documents [ds-atom]
(testing "create document with specific value and see if find-documents will find it"
(let [needle-attributes {:title (str (gensym "unique"))
:id (str (gensym "unique"))
:x 35000}
needle (-> (generate-document)
(merge needle-attributes))
haystack (repeatedly 200 #(-> (generate-document)
(assoc :id (str (gensym "random-id")))))
repo :testrepo]
(doseq [d haystack]
(.save-document @ds-atom (:id d) d repo))
(.save-document @ds-atom (:id needle) needle repo)
(is (= needle (.load-document @ds-atom (:id needle) repo)) "Check if testdocument was saved correctly")
(let [find-results (.find-documents @ds-atom [[:title [= (:title needle-attributes)]]
[:x [= (:x needle-attributes)]]] repo)]
(is (= 1 (count find-results)) "Search should return only one document")
(is (= needle (first (vals find-results))) "Search should return testdocument")))))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment