Browse Source

Implemented publishing and retrieving of webroots

started to play with blob handling

managed to publish/get blobs and read/post messages

Implemented uploading a webroot

major refactoring and documentation

downloading a webroot works now
Vincent Truchseß 10 months ago
5 changed files with 159 additions and 4 deletions
  1. +5
  2. +1
  3. +29
  4. +45
  5. +79

+ 5
- 3
project.clj View File

@@ -1,9 +1,11 @@
(defproject saljut-publish "0.1.0-SNAPSHOT"
(defproject saljut "0.1.0-SNAPSHOT"
:description "FIXME: write description"
:url ""
:license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"
:url ""}
:dependencies [[org.clojure/clojure "1.10.0"]]
:main ^:skip-aot saljut-publish.core
:dependencies [
[org.clojure/clojure "1.10.0"]
[org.clojure/data.json "0.2.6"]]
:main ^:skip-aot saljut.core
:target-path "target/%s"
:profiles {:uberjar {:aot :all}})

src/saljut_publish/core.clj → src/saljut/core.clj View File

@@ -1,4 +1,4 @@
(ns saljut-publish.core
(ns saljut.core

(defn -main

+ 29
- 0
src/saljut/io.clj View File

@@ -0,0 +1,29 @@
(:require [clojure.string :as st])
(:require [ :as io])

(defn slurp-bytes [file]
"Behaves like slurp but reads a file as bytes and returns a byte-array"
(with-open [in (io/input-stream file)
out (]
(io/copy in out)
(.toByteArray out)))

(defn spit-bytes [file x]
"Behaves like spit but takes a byte-array as argument"
(with-open [out (io/output-stream file)]
(.write out x)))
(defn relative-file [basedir file]
"returs a string representing the path tp file relative to basedir.
basedir MUST be a parent-directory of file"
(st/replace (str file) (str (io/file basedir) "/") ""))

(defn get-all-children [dir]
"returns a seq of all non-directory children as objects"
(apply concat (map #((if (.isDirectory (io/file %))
(.listFiles (io/file dir)))))

+ 45
- 0
src/saljut/sbot.clj View File

@@ -0,0 +1,45 @@
(ns saljut.sbot
(:require [ :as sh])
(:require [ :as sio])
(:require [saljut.util.sbot :as ut])
(:require [ :as io])

(defn get-latest-msg [pubkey domain]
"Get the most recent SSB-message of type \"webroot\" from the identity of
pubkey referring to domain's webroot"
(first (max-key :sequence
(filter #(= domain (:domain %))
(map ut/parse-msg
(filter ut/validate-msg?
(filter ut/webroot-msg? (ut/get-user-stream pubkey))))))))

(defn publish-manifest [domain manifest]
"Publishes a SSB-message of type \"webroot\" referring to domain and
containing manifest."
(if manifest
(apply sh/sh "ssb-server" "publish" "--type" "webroot" "--domain" domain
(ut/to-command-vec manifest))))

(defn upload-dir [dir]
"Upoads the content of dir as a series of SSB-blobs. Returns a manifest-map
same as :manifest in parse-msg."
(into {}
(map #(vector (sio/relative-file dir %)
(ut/upload-file %))
(sio/get-all-children dir))))

(defn publish-dir [dir domain]
"publishes a directory in SSB under domain"
(publish-manifest domain (upload-dir dir)))

(defn download-dir [manifest target]
"Downloads all files in manifest into target"
(let [n (count manifest)]
(doseq [[i [k v]] (map vector (range 1 (inc n)) manifest)]
(let [file (io/file target k)
parent (.getParent file)]
(if (not (.exists (io/file parent)))
(.mkdirs (io/file parent)))
(ut/download-file v (io/file target k))
(println (str "(" i "/" n ") " v " => " k))))))

+ 79
- 0
src/saljut/util/sbot.clj View File

@@ -0,0 +1,79 @@
(ns saljut.util.sbot
(:require [ :as sh])
(:require [clojure.string :as st])
(:require [ :as json])
(:require [ :as sio])

(defn add-blob [content]
"Uploads a blob into SSB. Returns the blob-id as string"
(let [return (sh/sh "ssb-server" "blobs.add" :in content)]
(if (zero? (:exit return))
(first (st/split-lines (:out return)))
(print (:err return)))))

(defn get-blob [blob-id]
"Retrieves a blob from SSB. Returns the blob's content as byte-array"
(let [return (sh/sh "ssb-server" "blobs.get" blob-id :out-enc :bytes)]
(if (zero? (:exit return))
(:out return)
(print (:err return)))))

(defn get-user-stream [pubkey]
"When pubkey is a valid ssb-identity, returns a seq of maps representing
the json-objects in the identity's message-stream. Json-keys are converted
to keywords"
(let [return (sh/sh "ssb-server" "createUserStream" "--id" pubkey)]
(if (zero? (:exit return))
(map #(json/read-str % :key-fn keyword)
(clojure.string/split (:out return) #"\n\n"))
(print (:err return)))))

(defn webroot-msg? [msg]
"Predicate-function returning true if msg is a map from a mesage-stream with
message-type \"webroot\"."
(= "webroot" (get-in msg [:value :content :type])))

(defn validate-msg? [msg]
"Predicate-function returning true if msg is a mpp containing the mandatory
content of a ssb-message of type \"webroot\"."
(let [content (get-in msg [:value :content])
domain (:domain content)
manifest (:manifest content)]
(and (and domain manifest)
(and (string? domain)
(map? manifest)
(and (every? map? (vals manifest))
(every? string? (map :path (vals manifest)))
(every? string? (map :id (vals manifest))))))))

(defn parse-msg [msg]
"Converts a SSB-message of type \"webroot\" into a map in hte following
:sequence => the message's sequence number
:domain => the webroot's subdomain
:manifest => a map representing the webroot's manifest like:
relative path to file as string => blob-id as string"
{:sequence (get-in msg [:value :sequence])
:domain (get-in msg [:value :content :domain])
:manifest (into {}
(map #(vector (:path %)
(:id %))
(vals (get-in msg [:value :content :manifest]))))})

(defn to-command-vec [manifest]
"Returns a vector of strings as commandline-arguments representing manifest's
data to be applied to a shell-call to ssb-server. manifest is a map same as
:manifest in parse-msg's output."
(concat (map (fn [[i [k v]]]
(vector (str "--manifest." i ".path") k
(str "--manifest." i ".id") v))
(zipmap (range) manifest))))

(defn upload-file [file]
"Uploads file as SSB-blob. Returns the blob-id as string"
(add-blob (sio/slurp-bytes file)))

(defn download-file [blob-id target]
"downloads a file referenced by blob-id from ssb and saves it to target"
(sio/spit-bytes target (get-blob blob-id)))