~rbn/packagetools

98c83381290f70ec32d023dfb9ca574a76c79719 — Ruben Schuller 1 year, 9 months ago 0.0.1
initial commit
5 files changed, 590 insertions(+), 0 deletions(-)

A LICENCE
A README.md
A info.go
A packages.go
A packagetools.go
A  => LICENCE +24 -0
@@ 1,24 @@
Copyright (c) 2018 Ruben Schuller <code@rbn.im>. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

A  => README.md +52 -0
@@ 1,52 @@
# packagetools

is a utility for slackware to extract information about packages
which can be used in a shell pipeline.

## installation

with go installed and `$GOPATH/bin` in your `$PATH` just run

	go install

and you'll have packagetools installed.

## usage of packagetools

	-available
	  	list repository packages
	-list
	  	list installed packages
	-local string
	  	directory of package install logs (default "/var/log/packages")
	-prefixes string
	  	package subdirs in repository to consider (default "./patches/packages ./slackware64")
	-repo string
	  	path to repository (only CHECKSUMS.md5 and FILELIST.TXT are required) (default "/mnt/mirror/slackware/slackware64-14.2")
	-upgrade
	  	list upgradeable packages

## output format

For -available and -list the output format is a tab-seperated list of
name, version, arch, build, unix time of modification, file path

For -upgrade the output is similar, only that the first six fields are
information about the local package and the last six fields information
about the repository package.

## usage examples

this assumes the default location for repositories is valid.

### find all packages which can be upgraded, except kernel packages, and print a download url for the package and the signature

	for x in $(packagetools -upgrade | grep -v "kernel" | cut -f 12); do
		echo "http://ftp.slackware.com/pub/slackware/slackware64-14.2/$x"
		echo "http://ftp.slackware.com/pub/slackware/slackware64-14.2/$x.asc"
	done

### search a specific package in the repository and display the name, version and build

	packagetools -available | grep "bash" | cut -f 1,2,4


A  => info.go +103 -0
@@ 1,103 @@
package main

import (
	"fmt"
	"time"
)

//info contains package information
type info struct {
	Name     string
	Version  string
	Arch     string
	Build    string
	ModTime  time.Time
	CheckSum string
	Path     string
}

func (i info) String() string {
	return fmt.Sprintf("%v\t%v\t%v\t%v\t%v\t%v", i.Name, i.Version, i.Arch, i.Build, i.ModTime.Unix(), i.Path)
}

func mergeInfos(a, b info) (out info, err error) {
	if a.Name != "" && b.Name == "" || a.Name == b.Name {
		out.Name = a.Name
	} else if a.Name == "" && b.Name != "" {
		out.Name = b.Name
	} else {
		return out, fmt.Errorf("Name conflict: a: %v b: %v", a.Name, b.Name)
	}

	if a.Version != "" && b.Version == "" || a.Version == b.Version {
		out.Version = a.Version
	} else if a.Version == "" && b.Version != "" {
		out.Version = b.Version
	} else {
		return out, fmt.Errorf("Version conflict: a: %v b: %v", a.Version, b.Version)
	}

	if a.Arch != "" && b.Arch == "" || a.Arch == b.Arch {
		out.Arch = a.Arch
	} else if a.Arch == "" && b.Arch != "" {
		out.Arch = b.Arch
	} else {
		return out, fmt.Errorf("Arch conflict: a: %v b: %v", a.Arch, b.Arch)
	}

	if a.Build != "" && b.Build == "" || a.Build == b.Build {
		out.Build = a.Build
	} else if a.Build == "" && b.Build != "" {
		out.Build = b.Build
	} else {
		return out, fmt.Errorf("Build conflict: a: %v b: %v", a.Build, b.Build)
	}

	if !a.ModTime.IsZero() && b.ModTime.IsZero() || a.ModTime == b.ModTime {
		out.ModTime = a.ModTime
	} else if a.ModTime.IsZero() && !b.ModTime.IsZero() {
		out.ModTime = b.ModTime
	} else {
		return out, fmt.Errorf("ModTime conflict: a: %v b: %v", a.ModTime, b.ModTime)
	}

	if a.CheckSum != "" && b.CheckSum == "" || a.CheckSum == b.CheckSum {
		out.CheckSum = a.CheckSum
	} else if a.CheckSum == "" && b.CheckSum != "" {
		out.CheckSum = b.CheckSum
	} else {
		return out, fmt.Errorf("CheckSum conflict: a: %v b: %v", a.CheckSum, b.CheckSum)
	}

	if a.Path != "" && b.Path == "" || a.Path == b.Path {
		out.Path = a.Path
	} else if a.Path == "" && b.Path != "" {
		out.Path = b.Path
	} else {
		return out, fmt.Errorf("Path conflict: a: %v b: %v", a.Path, b.Path)
	}

	return
}

type infoByName []info

func (s infoByName) Len() int           { return len(s) }
func (s infoByName) Less(i, j int) bool { return s[i].Name < s[j].Name }
func (s infoByName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

//upgradeInfo pairs information about locally installed packages and packages in the repository.
type upgradeInfo struct {
	Local info
	Repo  info
}

func (u upgradeInfo) String() string {
	return fmt.Sprintf("%v\t%v", u.Local, u.Repo)
}

type upgradeInfoByName []upgradeInfo

func (s upgradeInfoByName) Len() int           { return len(s) }
func (s upgradeInfoByName) Less(i, j int) bool { return s[i].Local.Name < s[j].Local.Name }
func (s upgradeInfoByName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

A  => packages.go +272 -0
@@ 1,272 @@
package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

//suffixes are common package file extensions
var suffixes = []string{".tgz", ".txz", ".tbz2", ".tar.gz", ".tar.xz", ".tar.bz2"}

//hasPrefixes tests s if it has any prefix of pre
func hasPrefixes(s string, pre ...string) bool {
	for _, v := range pre {
		if strings.HasPrefix(s, v) {
			return true
		}
	}

	return false
}

//hasSuffixes tests s if it has any suffix of suf
func hasSuffixes(s string, suf ...string) bool {
	for _, v := range suf {
		if strings.HasSuffix(s, v) {
			return true
		}
	}

	return false
}

//trimSuffixes removes any suffix in suf from s
//NB: it doesn't stop after the first removal.
func trimSuffixes(s string, suf ...string) string {
	for _, v := range suf {
		s = strings.TrimSuffix(s, v)
	}

	return s
}

//parsePackagePath parses a filename into package information.
func parsePackagePath(packagePath string) (info, error) {
	// basename of file, remove package suffixes so they don't end up in build, split at dashes
	x := strings.Split(trimSuffixes(filepath.Base(packagePath), suffixes...), "-")
	if len(x) < 3 {
		return info{}, fmt.Errorf("invalid package name with less than 3 dashes: %v", packagePath)
	}
	name := strings.Join(x[0:len(x)-3], "-")
	version := x[len(x)-3]
	arch := x[len(x)-2]
	build := x[len(x)-1]

	if name == "" || version == "" || arch == "" || build == "" {
		return info{}, fmt.Errorf("empty fields in package: %v", packagePath)
	}

	return info{Name: name, Version: version, Arch: arch, Build: build, Path: packagePath}, nil
}

//packageTimestamps reads FILELIST.TXT like contents found in slackware repositories.
//it extracts package paths and modification times. only file paths starting with one of prefixes are considered.
func packageTimestamps(filelist io.Reader, prefixes ...string) (map[string]info, error) {
	out := make(map[string]info)

	s := bufio.NewScanner(filelist)
	for s.Scan() {
		fs := strings.Fields(s.Text())
		if len(fs) != 8 {
			if debug {
				log.Println("timestamps: skipping line with field count not 8:", s.Text())
			}
			continue
		}

		// ignore non package lines (asc, txt, etc.)
		if !hasSuffixes(fs[7], suffixes...) {
			if debug {
				log.Println("timestamps: skipping line with no matching suffix in file:", s.Text())
			}
			continue
		}

		if fs[2] != "root" || fs[3] != "root" {
			if debug {
				log.Println("timestamps: skipping line with uid/gid not root:", s.Text())
			}
			continue
		}

		if len(prefixes) != 0 && !hasPrefixes(fs[7], prefixes...) {
			if debug {
				log.Println("timestamps: skipping line with no matching prefix in file:", s.Text())
			}
			continue
		}

		if _, ok := out[fs[7]]; ok {
			return nil, fmt.Errorf("duplicate entry in filelist: %v", fs[7])
		}

		i, err := parsePackagePath(fs[7])
		if err != nil {
			return nil, err
		}

		modTime, err := time.Parse("2006-01-02 15:04", fmt.Sprintf("%v %v", fs[5], fs[6]))
		if err != nil {
			return nil, err
		}

		i.ModTime = modTime

		out[fs[7]] = i
	}

	return out, nil
}

//packageChecksums reads CHECKSUMS.md5 like formatted contents found in slackware repositories.
//it extracts package information and the package checksum. only file paths starting with one of prefixes are considered.
func packageChecksums(checksums io.Reader, prefixes ...string) (map[string]info, error) {
	s := bufio.NewScanner(checksums)
	for s.Scan() {
		if strings.HasPrefix(s.Text(), "MD5 message digest") {
			break
		}
	}

	pkgs := make(map[string]info)
	for s.Scan() {
		fs := strings.Split(s.Text(), "  ")
		if len(fs) != 2 {
			return nil, fmt.Errorf("invalid CHECKSUMS.md5 line: %v", s.Text())
		}

		if len(prefixes) != 0 && !hasPrefixes(fs[1], prefixes...) {
			if debug {
				log.Println("checksums: skipping line with no matching prefix in file:", s.Text())
			}
			continue
		}

		// ignore non package lines (asc, txt, etc.)
		if !hasSuffixes(fs[1], suffixes...) {
			if debug {
				log.Println("checksums: skipping line with no matching suffix in file:", s.Text())
			}
			continue
		}

		i, err := parsePackagePath(fs[1])
		if err != nil {
			return nil, err
		}

		pkgs[fs[1]] = i
	}

	return pkgs, nil
}

//readRepository reads FILELIST.TXT and CHECKSUMS.md5, parsing their information into an info-slice.
func readRepository(repo string, prefixes []string) ([]info, error) {
	f, err := os.Open(filepath.Join(repo, "FILELIST.TXT"))
	if err != nil {
		return nil, err
	}

	tsPkgs, err := packageTimestamps(f, prefixes...)
	if err != nil {
		return nil, err
	}

	if err := f.Close(); err != nil {
		return nil, err
	}

	f, err = os.Open(filepath.Join(repo, "CHECKSUMS.md5"))
	if err != nil {
		return nil, err
	}

	csPkgs, err := packageChecksums(f, prefixes...)
	if err != nil {
		return nil, err
	}

	if err := f.Close(); err != nil {
		return nil, err
	}

	//merge the two maps, retaining the newest version of a package.
	newestPkgs := make(map[string]info)
	for k, v := range csPkgs {
		w, ok := tsPkgs[k]
		if !ok {
			return nil, fmt.Errorf("package in checksums not found in timestamps: %v", k)
		}

		//		mergedPkg := info{Name: v.Name, Version: v.Version, Arch: v.Arch, Build: v.Build, ModTime: w.ModTime, CheckSum: v.CheckSum, Path: v.Path}

		mergedPkg, err := mergeInfos(v, w)
		if err != nil {
			return nil, err
		}

		_, ok = newestPkgs[v.Name]
		if !ok {
			newestPkgs[v.Name] = mergedPkg
			continue
		}

		if newestPkgs[v.Name].ModTime.Before(mergedPkg.ModTime) {
			newestPkgs[v.Name] = mergedPkg
		}
	}

	out := []info{}
	for _, v := range newestPkgs {
		out = append(out, v)
	}

	sort.Sort(infoByName(out))

	return out, nil
}

type fileInfosByName []os.FileInfo

func (x fileInfosByName) Len() int           { return len(x) }
func (x fileInfosByName) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
func (x fileInfosByName) Less(i, j int) bool { return x[i].Name() < x[j].Name() }

//readPackageLog reads a local directory of package logs returning a info-slice.
func readPackageLog(packageLog string) ([]info, error) {
	pkgs := []info{}

	f, err := os.Open(packageLog)
	if err != nil {
		return nil, err
	}

	fileInfos, err := f.Readdir(0)
	if err != nil {
		return nil, err
	}

	// sort so we get packages alphabetically
	sort.Sort(fileInfosByName(fileInfos))

	for _, v := range fileInfos {
		i, err := parsePackagePath(v.Name())
		if err != nil {
			return nil, err
		}

		i.ModTime = v.ModTime()

		pkgs = append(pkgs, i)
	}

	return pkgs, nil
}

A  => packagetools.go +139 -0
@@ 1,139 @@
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
)

const outputUsage = `
Output format:
For -available and -list the output format is a tab-seperated list of
name, version, arch, build, unix time of modification, file path

For -upgrade the output is similar, only that the first six fields are
information about the local package and the last six fields information
about the repository package.
`

var debug = false

//available returns a slice of infos for packages in the repository
func available(slackwareDir string, prefixes []string) ([]info, error) {
	pkgs, err := readRepository(slackwareDir, prefixes)
	if err != nil {
		return nil, err
	}

	return pkgs, nil
}

//list locally installed packages
func list(packageLog string) ([]info, error) {
	is, err := readPackageLog(packageLog)
	if err != nil {
		return nil, err
	}

	resultIs := make([]info, len(is))
	i := 0
	for _, v := range is {
		resultIs[i] = v
		i++
	}

	return resultIs, nil
}

//upgrade matches locally installed packages to packages in the repository, selecting the newest package
//from the repository as match to a local package.
func upgrade(repo string, prefixes []string, local string) ([]upgradeInfo, error) {
	repoPkgs, err := readRepository(repo, prefixes)
	if err != nil {
		return nil, err
	}

	localPkgs, err := readPackageLog(local)
	if err != nil {
		return nil, err
	}

	out := []upgradeInfo{}
	for _, localPkg := range localPkgs {
		match := upgradeInfo{}
		for _, repoPkg := range repoPkgs {
			if repoPkg.Name == localPkg.Name {
				if repoPkg.Version != localPkg.Version || repoPkg.Build != localPkg.Build {
					match = upgradeInfo{Local: localPkg, Repo: repoPkg}
				}
				break
			}
		}

		if match == (upgradeInfo{}) {
			continue
		}

		out = append(out, match)
	}

	sort.Sort(upgradeInfoByName(out))
	return out, nil
}

func main() {
	repo := flag.String("repo", "/mnt/mirror/slackware/slackware64-14.2", "path to repository (only CHECKSUMS.md5 and FILELIST.TXT are required)")
	prefixesFlag := flag.String("prefixes", "./patches/packages ./slackware64", "package subdirs in repository to consider")
	local := flag.String("local", "/var/log/packages", "directory of package install logs")
	availableFlag := flag.Bool("available", false, "list repository packages")
	listFlag := flag.Bool("list", false, "list installed packages")
	upgradeFlag := flag.Bool("upgrade", false, "list upgradeable packages")

	flag.Usage = func() {
		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		fmt.Fprintf(flag.CommandLine.Output(), outputUsage)
	}

	flag.Parse()

	prefixes := strings.Split(*prefixesFlag, " ")

	var err error
	var fs []fmt.Stringer

	switch {
	case *availableFlag:
		is, err2 := available(*repo, prefixes)
		err = err2
		fs = make([]fmt.Stringer, len(is))
		for i, v := range is {
			fs[i] = fmt.Stringer(v)
		}
	case *listFlag:
		is, err2 := list(*local)
		err = err2
		fs = make([]fmt.Stringer, len(is))
		for i, v := range is {
			fs[i] = fmt.Stringer(v)
		}
	case *upgradeFlag:
		is, err2 := upgrade(*repo, prefixes, *local)
		err = err2
		fs = make([]fmt.Stringer, len(is))
		for i, v := range is {
			fs[i] = fmt.Stringer(v)
		}
	}

	if err != nil {
		log.Fatal(err)
	}

	for _, v := range fs {
		fmt.Println(v)
	}
}