~otheb/kb

bcf0eff8a214054aa3184ecf5da2878e12df8c7a — Olie Ayre 2 months ago e11c170
Wrote API server
A server/.dub/build/application-debug-linux.posix-x86_64-dmd_2089-812C1178274E04173527214DDE8C3CDE/server => server/.dub/build/application-debug-linux.posix-x86_64-dmd_2089-812C1178274E04173527214DDE8C3CDE/server +0 -0

A server/dub.json => server/dub.json +18 -0
@@ 0,0 1,18 @@
{
	"authors": [
		"Oliver Ayre"
	],
	"dependencies": {
		"lighttp": "~>0.5.3"
	},
	"description": "kb API server",
	"importPaths": [
		"."
	],
	"license": "AGPL-3.0",
	"name": "server",
	"sourceFiles": [
		"main.d"
	],
	"targetType": "executable"
}

A server/dub.selections.json => server/dub.selections.json +10 -0
@@ 0,0 1,10 @@
{
	"fileVersion": 1,
	"versions": {
		"libasync": "0.8.4",
		"lighttp": "0.5.3",
		"memutils": "0.4.13",
		"urld": "2.1.1",
		"xbuffer": "1.0.0"
	}
}

A server/example-config => server/example-config +8 -0
@@ 0,0 1,8 @@
{
	"port" : 8000 ,
	"data" : "./data" ,
	"tag-match" : 10 ,
	"tag-similar" : 5 ,
	"content-match" : 3 ,
	"content-similar" : 1
}

A server/libserver.a => server/libserver.a +0 -0

A server/main.d => server/main.d +336 -0
@@ 0,0 1,336 @@
module main ;

import std.algorithm ;
import std.file      ;
import std.json      ;
import std.conv      ;
import std.path      ;
import std.array     ;
import std.datetime  ;
import std.typecons  ;
import std.string    ;

import lighttp ;

// default config file
private const string CONFFILE = "/etc/kbs/config" ;
// port to listen on
private ushort PORT = 8000 ;
// directory containing data files
private string DATADIR = "./data" ;
// search heuristics
private long TAGMATCH       = 10 ;
private long TAGSIMILAR     =  5 ;
private long CONTENTMATCH   =  3 ;
private long CONTENTSIMILAR =  1 ;

int main( string[] args ) {
	import std.stdio : writeln ;
	// read config
	// if an argument is provided, that is used as the config file
	// config file will softly default to /etc/kbs/config
	if ( args.length > 2 ) {
		writeln( "Usage: " , args[0] , " [CONFIG FILE]" ) ;
		return 1 ;
	} else if ( args.length == 2 ) {
		if ( ! exists( args[1] ) ) {
			writeln( "Config file " , args[1] ,
			         " does not exist or cannot be found" ) ;
			return 1 ;
		} else if ( isDir( args[1] ) ) {
			writeln( "Config file " , args[1] , " is a directory" ) ;
			return 1 ;
		}
		readconf( args[1].readText() ) ;
	}
	if ( exists( CONFFILE ) ) //
	if ( isFile( CONFFILE ) ) { readconf( CONFFILE.readText() ) ; }

	// display config options on startup
	writeln( "Port: " , PORT ) ;
	writeln( "Data dir: " , DATADIR ) ;
	writeln( "Tag match heuristic: " , TAGMATCH ) ;
	writeln( "Tag similar heuristic: " , TAGSIMILAR ) ;
	writeln( "Content match heuristic: " , CONTENTMATCH ) ;
	writeln( "Content similar heuristic: " , CONTENTSIMILAR ) ;

	Server server = new Server() ;
	server.host( "0.0.0.0" , PORT ) ;
	server.host( "::" , PORT ) ;
	server.router.add( new router() ) ;
	server.run() ;
	return 0 ;
}

void readconf( string content ) {
	auto j = content.parseJSON() ;
	if ( ( "port" in j ) !is null ) { PORT = cast(ushort)j["port"].integer ; }
	if ( ( "data" in j ) !is null ) { DATADIR = j["data"].str ; }
	if ( ( "tag-match" in j ) !is null ) {
		TAGMATCH = j["tag-match"].integer ;
	}
	if ( ( "tag-similar" in j ) !is null ) {
		TAGSIMILAR = j["tag-similar"].integer ;
	}
	if ( ( "content-match" in j ) !is null ) {
		CONTENTMATCH = j["content-match"].integer ;
	}
	if ( ( "content-similar" in j ) !is null ) {
		CONTENTSIMILAR = j["content-similar"].integer ;
	}
}

class router {
	@Get( "" )
	index( ServerRequest req , ServerResponse res ) {
		res.contentType = MimeTypes.json ;
		res.body = JSONValue( [ "ids" : JSONValue(
			DATADIR.dirEntries( SpanMode.shallow ).map!( d =>
				d.name.baseName.to!ulong
			).array
		) ] ) ;
	}

	@Get( "e" , "[0-9]+" )
	get( ServerRequest req , ServerResponse res , string _id ) {
		entry e = read( _id ) ;
		if ( e is null ) { res.status = StatusCodes.notFound ;
		} else { res.body = e.stringify() ; }
	}

	@Delete( "e" , "[0-9]+" )
	delete_( ServerRequest req , ServerResponse res , string _id ) {
		string path = buildPath( DATADIR , _id ) ;
		if ( ! exists( path ) ) {
			res.status = StatusCodes.notFound ;
			return ;
		}
		remove( path ) ;
	}

	@Post( "e" )
	postnew( ServerRequest req , ServerResponse res ) {
		auto e = make( req.body ) ;
		e.id   = getnewid()                  ;
		e.time = Clock.currTime().toUnixTime ;
		e.write() ;
		res.contentType = MimeTypes.json ;
		res.body = JSONValue( [ "id" : JSONValue( e.id ) ] ) ;
	}

	@Post( "e" , "[0-9]+" )
	post( ServerRequest req , ServerResponse res , string _id ) {
		auto e = make( req.body ) ;
		entry a = read( _id ) ;
		if ( a is null ) {
			res.status = StatusCodes.notFound ;
			return ;
		}
		if ( a.old == e.old && isPermutation( a.tags , e.tags ) &&
		     a.content == e.content ) {
			// entries are identical
			res.status = StatusCodes.notModified ;
			return ;
		}
		a.old = e.old ;
		a.tags = e.tags ;
		if ( a.content != e.content ) {
			// shift history
			a.history = new hist( a.time , a.content ) ~ a.history ;
			if ( a.history.length > 64 ) {
				// truncate history to prevent files from getting too big
				a.history = a.history[0..64] ;
			}
			a.content = e.content ;
			a.time = Clock.currTime().toUnixTime ;
		}
		a.write() ;
		res.body = JSONValue( [ "id" : JSONValue( a.id ) ] ) ;
	}

	@CustomMethod( "PATCH" , true , [ "e" , "[0-9]+" ] )
	patch( ServerRequest req , ServerResponse res , string _id ) {
		auto e = make( req.body ) ;
		entry a = read( _id ) ;
		if ( a is null ) {
			res.status = StatusCodes.notFound ;
			return ;
		}
		if ( a.old == e.old && isPermutation( a.tags , e.tags ) &&
		     a.content == e.content ) {
			// entries are identical
			res.status = StatusCodes.notModified ;
			return ;
		}
		a.old     = e.old     ;
		a.tags    = e.tags    ;
		a.content = e.content ;
		a.time    = Clock.currTime().toUnixTime ;
		a.write() ;
		res.body = JSONValue( [ "id" : JSONValue( a.id ) ] ) ;
	}

	@Post( "s" )
	search( ServerRequest req , ServerResponse res ) {
		// get search terms
		string[] terms = req.body.parseJSON()["search"].str.tr( "A-Z" , "a-z" )
		                 .split() ;
		res.contentType = MimeTypes.json ;
		res.body = JSONValue( [ "results" : JSONValue(
			DATADIR.dirEntries( SpanMode.shallow )
			.map!( d => dosearch( d.name.baseName.to!long , terms ) )
			.filter!( r => r[1] > 0 ).map!( r => JSONValue( [
				"id"        : JSONValue( r[0] ) ,
				"relevance" : JSONValue( r[1] )
			] ) ).array
		) ] ) ;
	}
}

alias searchresult = Tuple!( long , long ) ;

searchresult dosearch( long i , string[] terms ) {
	searchresult result = tuple( i , 0 ) ;
	// get an entry, then search tags and content to generate relevance

	/*  RELEVANCE algorithm:

		The relevance algorithm provides a heuristic degree of relevance an
		entry has to a given set of search terms. A higher value of
		relevance equates to a post that should be sorted higher on a
		relevance list. Relevance is an integer value to allow easy sorting.

		If a search term exactly matches an entry tag, then relevance is
		increased by {TAGMATCH}, if a search term is part of a tag or a tag
		is part of a search term, each instance increases relevance by
		{TAGSIMILAR}. If a search term exactly matches a word in the entry
		content, relevance is increased by {CONTENTMATCH}, and if a search
		term appears inside the content, then each instance increases
		relevance by {CONTENTSIMILAR}.
	*/

	// assume that the entry exists
	entry e = read( i.to!string ) ;

	// for each term
	foreach ( string t ; terms ) {
		// check for tag matches and similarities
		foreach ( string tag ; e.tags.map!( m => m.tr( "A-Z" , "a-z" ) )
		          .array ) //
		if ( t == tag ) { result[1] += TAGMATCH ;
		} else if ( t.canFind( tag ) || tag.canFind( t ) ) {
			result[1] += TAGSIMILAR ;
		}

		// check for content matches and similarities
		string[] words = e.content.split() ;
		foreach ( string w ; words.map!( m => m.tr( "A-Z" , "a-z" ) )
		          .array ) //
		if ( w == t ) { result[1] += CONTENTMATCH ;
		} else if ( t.canFind( w ) || w.canFind( t ) ) {
			result[1] += CONTENTSIMILAR ;
		}
	}
	return result ;
}

private long getnewid() {
	// first get all dir entries
	long[] existing = DATADIR.dirEntries( SpanMode.shallow )
	                  .map!( d => d.name.baseName.to!long ).array ;
	for ( long i = 0 ; i < long.max ; i ++ ) //
	if ( ! existing.canFind( i ) ) { return i ; }
	// don't need to catch this because there's no way anyone is ever going to
	// reach this state in practice
	throw new Exception( "Run out of IDs!" ) ;
}

private class entry {
	long     id         ;
	string   title      ;
	long     time       ;
	bool     old        ;
	string[] tags       ;
	string   content    ;
	hist  [] history    ;

	this( long i , string t , long ts , bool o , string[] tg , string c ,
	      hist[] h ) {
		id         = i  ;
		title      = t  ;
		time       = ts ;
		old        = o  ;
		tags       = tg ;
		content    = c  ;
		history    = h  ;
	}

	this( string t , bool o , string[] tg , string c ) {
		title   = t  ;
		old     = o  ;
		tags    = tg ;
		content = c  ;
	}
}

private class hist {
	long   time    ;
	string content ;

	this( long t , string c ) {
		time    = t ;
		content = c ;
	}
}

private entry read( string id ) {
	string path = buildPath( DATADIR , id ) ;
	if ( ! exists( path ) ) { return null ;
	} else if ( ! isFile( path ) ) { return null ; }
	return path.readText().make() ;
}

private entry makepart( string data ) {
	auto j = parseJSON( data ) ;
	return new entry(
		( "title" in j ) !is null ? j["title"].str     : ""    ,
		( "old"   in j ) !is null ? j["old"]  .boolean : false ,
		( "tags"  in j ) !is null ?
			j["tags"].array.map!( t => t.str ).array : [] ,
		j["content"].str
	) ;
}

private entry make( string data ) {
	auto j = parseJSON( data ) ;
	return new entry(
		j["id"]        .integer  ,
		j["title"]     .str      ,
		j["time"]      .integer  ,
		j["old"]       .boolean  ,
		j["tags"]      .array.map!( t => t.str ).array ,
		j["content"]   .str      ,
		j["history"]   .array.map!( h =>
			new hist( h["time"].integer , h["content"].str ) ).array
	) ;
}

private void write( entry e ) {
	string path = buildPath( DATADIR , e.id.to!string ) ;
	std.file.write( path , e.stringify() ) ;
}

private string stringify( entry e ) {
	return JSONValue( [
		"id"         : JSONValue( e.id         ) ,
		"title"      : JSONValue( e.title      ) ,
		"time"       : JSONValue( e.time       ) ,
		"old"        : JSONValue( e.old        ) ,
		"tags"       : JSONValue( e.tags.map!( t => JSONValue( t ) ).array ) ,
		"content"    : JSONValue( e.content    ) ,
		"history"    : JSONValue( e.history.map!( h => JSONValue( [
			"time"    : JSONValue( h.time    ) ,
			"content" : JSONValue( h.content )
		] ) ).array )
	] ).toString() ;
}

A server/server => server/server +0 -0