fix find-documents in git storage, add versioned-repository? method

src/document_storage/git/core.clj

@@ -111,7 +111,11 @@
                 :cache (atom {})))
       (log/error "Unable to initialize Document Storage - no config!")))
   (list-repositories [this]
-    @repositories)
+    @repositories
+    (mapv
+     (fn [[id data]]
+       (assoc data :id id))
+     @repositories))
   (load-document [this id repository]
     (let [cached-repository (cached-repository? repository)]
       (or
@@ -160,16 +164,16 @@
       files))
 
   (find-documents [this query repository]
-    (let [docs (.list-documents this :repository repository)
+    (let [docs (.list-documents this repository)
           cached-repository (cached-repository? repository)]
       (or (when cached-repository
             (when-let [cached-query (get-in @cache [repository :queries query])]
               (into {} (map (fn [fname]
-                              (let [doc (.load-document this fname :repository repository)]
+                              (let [doc (.load-document this fname repository)]
                                 [fname doc]))
                             cached-query))))
           (let [result (into {} (keep (fn [fname]
-                                        (let [doc (.load-document this fname :repository repository)]
+                                        (let [doc (.load-document this fname repository)]
                                           (when (query-fits? doc query)
                                             [fname doc])))
 
@@ -181,6 +185,9 @@
     (pmap (fn [[id repo]]
             (push-upstreams id)) @repositories))
 
+  (versioned-repository? [this repository]
+    (not (no-commit-repository? repository)))
+
   (list-versions [this id repository]
     (let [repo (get-repo repository)]
       (sort-by :date (versioning/list-commits repo id))))

src/document_storage/git/versioning.clj

@@ -6,15 +6,19 @@
 
 
 (defn list-commits [repo path]
-    (-> repo
-        (jp/git-log :paths path)
-        (->>
-         (map #(hash-map
-               :commit (:id %)
-               :date (get-in % [:committer :date])
-               :version (jp/get-blob repo (:id %) path)))
-         (sort-by :date)
-         reverse)))
+  (-> repo
+      (jp/git-log :paths path)
+      (->>
+       (map #(let [blob (jp/get-blob repo (:id %) path)]
+               (hash-map
+                :commit (:id %)
+                :date (get-in % [:committer :date])
+                :version (jp/get-blob repo (:id %) path)
+                :type (if blob
+                        :update
+                        :delete))))
+       (sort-by :date)
+       reverse)))
 
 (defn load-revision [commit repo path]
   (let [git-repo   (.getRepository repo)

src/document_storage/postgres/core.clj

@@ -3,6 +3,7 @@
             [clojure.java.io :as io]
             [clojure.edn :refer [read-string]]
             [clojure.set :refer [difference]]
+            [clojure.string :as str]
             [fipp.edn :refer [pprint] :rename {pprint fipp}]
             [document-storage.protocol :as protocol]
             [document-storage.postgres.db :as db]
@@ -12,6 +13,13 @@
             [conman.core :as conman]
             [clj-helper.string :refer [get-unique-id]]))
 
+(defn prepare-repository-name [repo-id]
+  (-> (if (keyword? repo-id)
+        (name repo-id)
+        (str repo-id))
+      (str/replace "-" "_")
+      ))
+
 (defprotocol db-storage
   "Base Protocol for document storages"
   (get-schema [this] "Return the db-schema name")
@@ -30,9 +38,7 @@
   (run-query [this query-key params]
     (let [storage-schema (.get-schema this)
           repository (:repository params)
-          repository-name (if (keyword? repository)
-                            (name repository)
-                            (str repository))
+          repository-name (prepare-repository-name repository)
           table-names {:schema-name storage-schema
                        :repository-name repository-name
                        :ds-encoding-fqn (str storage-schema
@@ -56,7 +62,7 @@
   (init-storage [this]
     (log/info "Init Postgres-Storage.")
     (.run-query this :create-storage! {})
-    (let [repositories (set (map name (get (.-params this) :repositories [])))
+    (let [repositories (set (map prepare-repository-name (get (.-params this) :repositories [])))
           existing-repos (set (map :id (.list-repositories this)))]
       (log/info "Repositories: " repositories)
       (log/info "Existing Repositories: " existing-repos)
@@ -74,7 +80,7 @@
     (count (documents/load-document this id repository)))
 
   (save-document [this id document repository options]
-    (documents/save-document this id document repository))
+    (documents/save-document this id document repository options))
 
   (save-document [this id document repository]
     (.save-document this id document repository {}))
@@ -87,10 +93,14 @@
           (map :document_id (run-query this :get-all-document-ids
                                        {:repository repository}))))
 
+  (versioned-repository? [this repository]
+    ;;;TODO check if repo is versioned
+    true)
+
   (find-documents [this query repository]
-    (let [docs (.list-documents this :repository repository)]
+    (let [docs (.list-documents this repository)]
       (let [result (into {} (keep (fn [fname]
-                                    (let [doc (.load-document this fname :repository repository)]
+                                    (let [doc (.load-document this fname repository)]
                                       (when (query-fits? doc query)
                                         [fname doc])))
 

src/document_storage/postgres/documents.clj

@@ -27,7 +27,7 @@
 
 (defn restore-document [storage frame-id repository]
   (when-let [frame (get-document-frame-by-id storage frame-id repository)]
-    (case (:data_type frame)
+    (case (name (:data_type frame))
       "key" (encoding/decode-document (:data frame))
       "diff" (let [data (encoding/decode-document (:data frame))]
                (escript/patch
@@ -47,15 +47,16 @@
    (when-let [frame (get-document-frame-by-version storage document-id version repository)]
      (restore-document storage (:id frame) repository))))
 
-(defn save-keyframe [storage document-id document repository]
-  (let [doc (encoding/encode-document document)]
+(defn save-keyframe [storage document-id document last-version repository]
+  (let [doc (encoding/encode-document document)
+        last-version (or last-version 0)]
     ;;VALUES (:uuid, :document-id, :type, :encoding, :version, :reference, :data)
     (.run-query storage :create-document-frame!
                 {:repository repository
                  :document-id document-id
                  :type :ds-type/key
                  :encoding :ds-encoding/nippy
-                 :version 0
+                 :version (inc last-version)
                  :reference nil
                  :data doc})))
 
@@ -63,7 +64,7 @@
 (defn save-pframe [storage document-id doc last-version repository]
   (let [last-doc (load-document storage document-id last-version repository)
         last-frame (get-document-frame-by-version storage document-id last-version repository)
-        diff (escript/diff last-doc doc)
+        diff (escript/diff last-doc doc {:str-diff? false :algo :quick})
         diff-doc (encoding/encode-document (escript/get-edits diff))]
     (.run-query storage :create-document-frame!
                 {:repository repository
@@ -74,10 +75,12 @@
                  :reference (:id last-frame) ;;;FIXME save ID in reference not version
                  :data diff-doc})))
 
-(defn save-document [storage document-id document repository]
+(defn save-document [storage document-id document repository options]
   (if-let [last-version (get-last-document-version storage document-id repository)]
-    (save-pframe storage document-id document last-version repository)
-    (save-keyframe storage document-id document repository)))
+    (if (true? (:force-keyframe options))
+      (save-keyframe storage document-id document last-version repository)
+      (save-pframe storage document-id document last-version repository))
+    (save-keyframe storage document-id document -1 repository)))
 
 
 (defn delete-document [storage document-id repository]

src/document_storage/protocol.clj

@@ -10,6 +10,7 @@
   (list-repositories [ds] "List all repositories")
   (list-versions [ds id repository] "List all version of a specific document in a repository. The result is expected to be sorted in ascending order.")
   (load-version [ds id version repository] "Load a specific document version in a repository")
+  (versioned-repository? [ds repository] "Check if repository is versioned")
   (get-document-size [ds id repository] "Calculate the current document size")
   (init-storage [ds] "Start a document storage")
   (shutdown [ds] "Stop a document storage"))

test/document_storage/git/core_test.clj

@@ -44,6 +44,8 @@
 (deftest versioning-test
   (protocol-tests/test-versioning ds-atom))
 
+(deftest find-document-test
+  (protocol-tests/test-find-documents ds-atom))
 
 (deftest views-test
   (testing "save views, save basic structure, load it and load views"

test/document_storage/postgres/core_test.clj

@@ -27,3 +27,6 @@
 
 (deftest versioning-test
     (protocol-tests/test-versioning ds-atom))
+
+(deftest find-document-test
+  (protocol-tests/test-find-documents ds-atom))

test/document_storage/protocol_test.clj

@@ -46,3 +46,22 @@
           (is (= (diff structure-v2 structure-v3)
                  (diff storage-doc-v2 storage-doc-v3))))))))
 
+(defn test-find-documents [ds-atom]
+  (testing "create document with specific value and see if find-documents will find it"
+    (let [needle-attributes {:title (str (gensym "unique"))
+                             :id (str (gensym "unique"))
+                             :x 35000}
+          needle (-> (generate-document)
+                     (merge needle-attributes))
+          haystack (repeatedly 200 #(-> (generate-document)
+                                        (assoc :id (str (gensym "random-id")))))
+          repo :testrepo]
+      (doseq [d haystack]
+        (.save-document @ds-atom (:id d) d repo))
+      (.save-document @ds-atom (:id needle) needle repo)
+      (is (= needle (.load-document @ds-atom (:id needle) repo)) "Check if testdocument was saved correctly")
+      (let [find-results (.find-documents @ds-atom [[:title [= (:title needle-attributes)]]
+                                                    [:x [= (:x needle-attributes)]]] repo)]
+        (is (= 1 (count find-results)) "Search should return only one document")
+        (is (= needle (first (vals find-results))) "Search should return testdocument")))))
+