~otheb/kb

118d5bcf63e12b142ec51708755e9e515864462f — Olie Ayre 2 months ago 84b5be4
Fixed API urls and added title searching

Also updated readme with usage information for CLI to be written.
5 files changed, 314 insertions(+), 37 deletions(-)

M README
A cli/dub.json
A cli/kb.sh
A cli/main.d
M server/main.d
M README => README +33 -22
@@ 10,17 10,6 @@
   working simply with flat files and only requiring a small number of API
   functions to use.

The 'server' directory contains the core API server which acts as a proxy to the
file storage. The 'web' directory contains a simple server-side web UI (which
proudly runs without a byte of JavaScript). The 'cli' directory contains a CLI
client to `kb` for those that would like some RAM left with which to do some
work.

The 'server' directory contains the core API server and web UI (which proudly
runs without a byte of JavaScript), and the 'cli' directory contains a simple
CLI client with which to interact with the server in the likely case you'd
rather not have to use a browser.

# HOW IT WORKS

The data directory contains a collection of json files. Each file name is the


@@ 46,14 35,18 @@ properly handle authentication. That is beyond the scope of the project though.

You need but 7 simple requests to use `kb`:

GET /
	List all entry IDs. Returns 'application/json':
	{ ids : [ 0 , 1 , 7 ] }

GET /$n
	Get the content of an entry. Returns 'application/json':
	{
		id : 123 , // unique numeric id of the entry
		title : "The title" , // entry title
		time : 12345678 , // unix timestamp for the entry
		popularity : 8.4 , // popularity heuristic
		old : false , // boolean deprecation flag
		tags : [ "some" , "helpful" , "tags" ] , // tags for searching
		content : "large string body" ,
		history : [
			{


@@ 112,16 105,34 @@ POST /s
	number of tag matches, content matches, and popularity. Higher is considered
	better.

POST /b/$n
	Boost an entry to improve its popularity rating. Returns 'application/json'
	containing the new popularity:
	{ popularity : 8 }
	If the entry doesn't exist, returts a 404.
# CLI

A handful of commands make the CLI easy:

$ kb ls [-r]
	Generates a formatted listing of all entries and opens them in $PAGER
	(defaults to `less`). Sorts by ID, or `-r` can be provided to sort by
	timestamp, with more recently created/modified entries at the top.

$ kb view ID
	Opens a formatted view of the entry #ID in $PAGER.

$ kb new
	Opens a formatted template in $EDITOR (defaults to `vim`) to fill in
	information, then uses the resulting buffer to generate a new entry.

$ kb edit ID
	Opens a formatted template in $EDITOR pre-populated with the content in
	entry #ID for editing. An additional flag is present that allows specifying
	whether this is a new version or just a patch/fixup.

# USAGE
$ kb rm ID
	Deletes entry #ID

	API SERVER
$ kb search TERMS...
	Completes a search for entries based on the provided terms and displays
	results in the same manner as `kb ls`.

Build:
	cd server
	dub build
The very first argument to `kb` should be the hostname/IP and port of the API
server. Users should create a shell alias to include this information
automatically. `kb` itself does not have a config file.

A cli/dub.json => cli/dub.json +18 -0
@@ 0,0 1,18 @@
{
	"authors": [
		"Oliver Ayre"
	],
	"dependencies": {
		"lighttp": "~>0.5.3"
	},
	"description": "kb CLI client",
	"importPaths": [
		"."
	],
	"license": "AGPL-3.0",
	"name": "kb",
	"sourceFiles": [
		"main.d"
	],
	"targetType": "executable"
}

A cli/kb.sh => cli/kb.sh +106 -0
@@ 0,0 1,106 @@
#!/bin/bash

url="127.0.0.1:80"
pager="${PAGER:-less -r}"
editor="${EDITOR:-vim}"

# quick arg count check
[[ "$#" -lt 2 ]] && goto usage

# set host and port
url="http://$1"

# functions
usage() {
	echo 'Usage: '$0' <host> <command> [command-args...]
	host : hostname/IP/port of API server e.g. '\'myapiserver:8000\''
	commands :
	ls [-r] : list entries by ID (or by timestamp if `-r` is provided)
	view <id> : view entry #ID in $PAGER (defaults to `less -r`)
	new : create a new entry in $EDITOR (defaults to `vim`)
	edit <id> : edit entry #ID in $EDITOR
	rm <id> : delete entry #ID
	search <terms...> : search based on provided terms and display results
	in $PAGER'
	exit 1
}

formatdate() {
	# $1 is unix timestamp
	date "+%-d %b %Y @ %I:%M%p" -d @$1
}

formatpost() {
	# $1 is parsed json data
	local data="$1"
	echo -en "\x1b[34m#$(echo "$data" | sed '1q;d')\x1b[0m"
	[[ "$(echo "$data" | sed '2q;d')" == "true" ]] && \
		echo -en " [DEPRECATED]"
	echo -en " $(echo "$data" | sed '3q;d' | sed 's/^"//;s/"$//' )"
	echo " - $(formatdate "$(echo "$data" | sed '4q;d')")"
	echo -e "\tTags: $(echo "$data" | sed '1,4d' | xargs)"
}

dols() {
	curl --fail "$url/" 2> /dev/null | jq '.ids[]' | sort | while read id; do
		formatpost "$(curl "$url/$id" 2> /dev/null | \
			jq '.id , .old , .title , .time , .tags[]')"
	done | $pager
}

doview() {
	local data="$(curl --fail "$url/$1" 2> /dev/null)"
	[[ "$data" == "" ]] && return
	formatpost "$(echo "$data" | jq '.id , .old , .title , .time , .tags[]')"
	local content="$(echo "$data" | jq '.content')"
	echo "${content:1:-1}"
	echo -e "\nHistory:"
	echo "$data" | jq '.history[]'
}

donew() {
	echo "new"
}

doedit() {
	echo "edit $1"
}

dorm() {
	echo "rm $1"
}

dosearch() {
	echo "search $@"
}

# do command
case "$2" in
	ls)
		if [[ "$#" -eq 3 && "$3" == "-r" ]]; then dols true
		elif [[ "$#" -eq 2 ]]; then dols false
		else usage
		fi ;;
	view)
		if [[ "$#" -eq 3 && "$(echo -n $3 | tr -d 0-9)" == "" ]]; then doview $3
		else usage
		fi ;;
	new)
		if [[ "$#" -eq 2 ]]; then donew
		else usage
		fi ;;
	edit)
		if [[ "$#" -eq 3 && "$(echo -n $3 | tr -d 0-9)" == "" ]]; then doedit $3
		else usage
		fi ;;
	rm)
		if [[ "$#" -eq 3 && "$(echo -n $3 | tr -d 0-9)" == "" ]]; then dorm $3
		else usage
		fi ;;
	search)
		if [[ "$#" -gt 2 ]]; then dosearch "${@:3}"
		else usage
		fi ;;
	*) usage
esac
exit 0

A cli/main.d => cli/main.d +117 -0
@@ 0,0 1,117 @@
module main ;

import std.stdio  ;
import std.conv   ;
import std.array  ;
import std.string ;

import lighttp ;

int usage() {
	writeln(
"Usage: kb <host> <command> [command-args...]\n
\thost: hostname/IP/port of API server e.g. 'myapiserver:8000'
\tcommans:\n
\tls [-r] : list entries by ID (or by timestamp if `-r` is provided).
\tview ID : view entry #ID in $PAGER (defaults to `less`).
\tnew     : create a new entry in $EDITOR (defaults to `vim`).
\tedit ID : edit entry #ID in $EDITOR.
\trm ID   : delete entry #ID.
\tsearch TERMS... :
\t    search based on provided terms and display results in $PAGER." ) ;
	return 1 ;
}

string HOST   = "127.0.0.1" ;
ushort PORT   = 80          ;
string PAGER  = "less"      ;
string EDITOR = "vim"       ;

Client.ClientConnection client ;

int main( string[] args ) {
	if ( args.length < 3 ) { return usage() ; }

	{ // set host and port
		string[] hp = args[1].split( ":" ) ;
		HOST = hp[0] ;
		if ( hp.length > 1 ) { PORT = hp[1].to!ushort ; }
	}

	// connect
	client = new Client().connect( HOST , PORT ) ;

	// main command
	try //
	switch ( args[2] ) {
	case "ls" :
		if ( args.length == 4 && args[3] == "-r" ) { ls( true ) ;
		} else if ( args.length == 3 ) { ls( false ) ;
		} else { return usage() ; }
		break ;
	case "view" :
		if ( args.length == 4 && args[3].tr( "0-9" , "" , "d" ).length == 0 ) {
			view( args[3].to!long ) ;
		} else { return usage() ; }
		break ;
	case "new" :
		if ( args.length == 3 ) { new_() ;
		} else { return usage() ; }
		break ;
	case "edit" :
		if ( args.length == 4 && args[3].tr( "0-9" , "" , "d" ).length == 0 ) {
			edit( args[3].to!long ) ;
		} else { return usage() ; }
		break ;
	case "rm" :
		if ( args.length == 4 && args[3].tr( "0-9" , "" , "d" ).length == 0 ) {
			rm( args[3].to!long ) ;
		} else { return usage() ; }
		break ;
	case "search" :
		if ( args.length > 3 ) { search( args[3..$] ) ;
		} else { return usage() ; }
		break ;
	default : return usage() ;
	} catch ( Exception e ) {
		writeln( "Error: " , e.message ) ;
		return 1 ;
	}

	return 0 ;
}

string url() { return "http://" ~ HOST ~ ":" ~ PORT.to!string ; }

void ls( bool recent ) {
	writeln( "listing entries from http://" , HOST , ":" , PORT , " using " ,
	         PAGER , " and sorting them based on " ,
	         recent ? "timestamp." : "id." ) ;
	client
		.success( r => writeln( r.body ) )
		.failure( () => writeln( "no connection to API server at " , url ) )
		.get( "/" ).w ;
	writeln( "made the request" ) ;

}

void view( long id ) {
	writeln( "viewing entry " , id , " in " , PAGER ) ;
}

void new_() {
	writeln( "creating a new entry with " , EDITOR ) ;
}

void edit( long id ) {
	writeln( "editing entry " , id , " in " , EDITOR ) ;
}

void rm( long id ) {
	writeln( "deleting entry " , id ) ;
}

void search( string[] terms ) {
	writeln( "searching entries based on search terms " , terms ,
	         " and displaying results in " , PAGER ) ;
}

M server/main.d => server/main.d +40 -15
@@ 9,6 9,7 @@ import std.array     ;
import std.datetime  ;
import std.typecons  ;
import std.string    ;
import std.stdio : writeln ;

import lighttp ;



@@ 21,6 22,8 @@ private string DATADIR = "./data" ;
// search heuristics
private long TAGMATCH       = 10 ;
private long TAGSIMILAR     =  5 ;
private long TITLEMATCH     =  5 ;
private long TITLESIMILAR   =  3 ;
private long CONTENTMATCH   =  3 ;
private long CONTENTSIMILAR =  1 ;



@@ 47,12 50,14 @@ int main( string[] args ) {
	if ( isFile( CONFFILE ) ) { readconf( CONFFILE.readText() ) ; }

	// display config options on startup
	writeln( "Port: " , PORT ) ;
	writeln( "Data dir: " , DATADIR ) ;
	writeln( "Tag match heuristic: " , TAGMATCH ) ;
	writeln( "Tag similar heuristic: " , TAGSIMILAR ) ;
	writeln( "Content match heuristic: " , CONTENTMATCH ) ;
	writeln( "Content similar heuristic: " , CONTENTSIMILAR ) ;
	writeln( "                     Port : " , PORT ) ;
	writeln( "                 Data dir : " , DATADIR ) ;
	writeln( "      Tag match heuristic : " , TAGMATCH ) ;
	writeln( "    Tag similar heuristic : " , TAGSIMILAR ) ;
	writeln( "    Title match heuristic : " , TITLEMATCH ) ;
	writeln( "  Title similar heuristic : " , TITLESIMILAR ) ;
	writeln( "  Content match heuristic : " , CONTENTMATCH ) ;
	writeln( "Content similar heuristic : " , CONTENTSIMILAR ) ;

	Server server = new Server() ;
	server.host( "0.0.0.0" , PORT ) ;


@@ 72,6 77,12 @@ void readconf( string content ) {
	if ( ( "tag-similar" in j ) !is null ) {
		TAGSIMILAR = j["tag-similar"].integer ;
	}
	if ( ( "title-match" in j ) !is null ) {
		TITLEMATCH = j["title-match"].integer ;
	}
	if ( ( "title-similar" in j ) !is null ) {
		TITLESIMILAR = j["title-similar"].integer ;
	}
	if ( ( "content-match" in j ) !is null ) {
		CONTENTMATCH = j["content-match"].integer ;
	}


@@ 83,6 94,7 @@ void readconf( string content ) {
class router {
	@Get( "" )
	index( ServerRequest req , ServerResponse res ) {
		writeln( "getting index" ) ;
		res.contentType = MimeTypes.json ;
		res.body = JSONValue( [ "ids" : JSONValue(
			DATADIR.dirEntries( SpanMode.shallow ).map!( d =>


@@ 91,15 103,17 @@ class router {
		) ] ) ;
	}

	@Get( "e" , "[0-9]+" )
	@Get( "([0-9]+)" )
	get( ServerRequest req , ServerResponse res , string _id ) {
		writeln( "getting entry " , _id ) ;
		entry e = read( _id ) ;
		if ( e is null ) { res.status = StatusCodes.notFound ;
		} else { res.body = e.stringify() ; }
	}

	@Delete( "e" , "[0-9]+" )
	@Delete( "([0-9]+)" )
	delete_( ServerRequest req , ServerResponse res , string _id ) {
		writeln( "deleting entry " , _id ) ;
		string path = buildPath( DATADIR , _id ) ;
		if ( ! exists( path ) ) {
			res.status = StatusCodes.notFound ;


@@ 108,9 122,10 @@ class router {
		remove( path ) ;
	}

	@Post( "e" )
	@Post( "" )
	postnew( ServerRequest req , ServerResponse res ) {
		auto e = make( req.body ) ;
		writeln( "making new entry" ) ;
		auto e = makepart( req.body ) ;
		e.id   = getnewid()                  ;
		e.time = Clock.currTime().toUnixTime ;
		e.write() ;


@@ 118,9 133,10 @@ class router {
		res.body = JSONValue( [ "id" : JSONValue( e.id ) ] ) ;
	}

	@Post( "e" , "[0-9]+" )
	@Post( "([0-9]+)" )
	post( ServerRequest req , ServerResponse res , string _id ) {
		auto e = make( req.body ) ;
		writeln( "adding new version of entry " , _id ) ;
		auto e = makepart( req.body ) ;
		entry a = read( _id ) ;
		if ( a is null ) {
			res.status = StatusCodes.notFound ;


@@ 148,8 164,9 @@ class router {
		res.body = JSONValue( [ "id" : JSONValue( a.id ) ] ) ;
	}

	@CustomMethod( "PATCH" , true , [ "e" , "[0-9]+" ] )
	@CustomMethod( "PATCH" , true , [ "([0-9]+)" ] )
	patch( ServerRequest req , ServerResponse res , string _id ) {
		writeln( "patching entry " , _id ) ;
		auto e = make( req.body ) ;
		entry a = read( _id ) ;
		if ( a is null ) {


@@ 175,6 192,7 @@ class router {
		// get search terms
		string[] terms = req.body.parseJSON()["search"].str.tr( "A-Z" , "a-z" )
		                 .split() ;
		writeln( "performing search with terms " , terms ) ;
		res.contentType = MimeTypes.json ;
		res.body = JSONValue( [ "results" : JSONValue(
			DATADIR.dirEntries( SpanMode.shallow )


@@ 222,10 240,17 @@ searchresult dosearch( long i , string[] terms ) {
			result[1] += TAGSIMILAR ;
		}

		// check for title matches and similarities
		string[] titlewords = e.title.split() ;
		foreach ( string w ; titlewords.map!( m => m.toLower() ) ) //
		if ( w == t ) { result[1] += TITLEMATCH ;
		} else if ( t.canFind( w ) || w.canFind( t ) ) {
			result[1] += TITLESIMILAR ;
		}

		// check for content matches and similarities
		string[] words = e.content.split() ;
		foreach ( string w ; words.map!( m => m.tr( "A-Z" , "a-z" ) )
		          .array ) //
		foreach ( string w ; words.map!( m => m.tr( "A-Z" , "a-z" ) ).array ) //
		if ( w == t ) { result[1] += CONTENTMATCH ;
		} else if ( t.canFind( w ) || w.canFind( t ) ) {
			result[1] += CONTENTSIMILAR ;