~louis77/pg2sqlite

d77f7c776a6a9c2bb6f2c4645fd7c962b6da0085 — Louis Brauer 2 months ago 90a8f07
Add --ignore-columns
8 files changed, 111 insertions(+), 54 deletions(-)

M README.md
M go.mod
M go.sum
M main.go
M pg.go
M sqlite.go
M ui.go
A utils.go
M README.md => README.md +16 -12
@@ 31,6 31,7 @@ Options:
      --pg-url                *Postgres connection string (i.e. postgres://localhost:5432/mydb)
      --sqlite-file           *Path to SQLite database file (i.e. mydatabase.db)
      --table                 *Name of table to export
      --ignore-columns         comma-separated list of columns to ignore
      --drop-table-if-exists   DANGER: Drop target table if it already exists
```



@@ 39,18 40,19 @@ Options:
```shell
$ pg2sqlite --pg-url postgres://localhost:5432/defaultdb \
            --sqlite-file mysqlite.db \
            --table daily_sales
            --table daily_sales \
            --ignore-columns raw_hash

Schema of table "daily_sales"
Column                     | Type                    
-------------------------- | ------------------------
reference_id               | integer                 
checkin                    | date                    
checkout                   | date                    
price                      | numeric                 
currency                   | character               
ratecode                   | character varying       
ts                         | timestamp with time zone
Column                     | Type                     | Ignore
-------------------------- | ------------------------ | ------
reference_id               | integer                  | No
checkin                    | date                     | No
checkout                   | date                     | No
price                      | numeric                  | No
raw_hash                   | character                | Yes
currency                   | character                | No
ts                         | timestamp with time zone | No
             
Creating Table statement:
CREATE TABLE daily_sales (         


@@ 58,12 60,14 @@ CREATE TABLE daily_sales (
        checkin TEXT, 
        checkout TEXT, 
        price REAL, 
        currency TEXT, 
        ratecode TEXT, 
        currency TEXT,  
        ts TEXT )
Does this look ok? (Y/N) y

Estimated row count: 50042260
Loading data with this statement:
SELECT "reference_id", "checkin", "checkout", "price", "currency", "external_id", "ts" FROM results_y2021m02 T

  24s [==>-----------------------------------------------------------------]   3%

Finished.

M go.mod => go.mod +7 -0
@@ 5,7 5,14 @@ go 1.16
require (
	github.com/gosuri/uilive v0.0.4 // indirect
	github.com/gosuri/uiprogress v0.0.1
	github.com/jackc/pgproto3/v2 v2.1.0 // indirect
	github.com/jackc/pgx/v4 v4.11.0
	github.com/mattn/go-colorable v0.1.8 // indirect
	github.com/mattn/go-isatty v0.0.13 // indirect
	github.com/mattn/go-sqlite3 v1.14.7
	github.com/mkideal/cli v0.2.5
	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
	golang.org/x/text v0.3.6 // indirect
)

M go.sum => go.sum +15 -0
@@ 153,6 153,8 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:
github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.0.6 h1:b1105ZGEMFe7aCvrT1Cca3VoVb4ZFMaFJLJcg/3zD+8=
github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.1.0 h1:h2yg3kjIyAGSZKDijYn1/gXHlYLCwl9ZjEh2PU0yVxE=
github.com/jackc/pgproto3/v2 v2.1.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=


@@ 210,6 212,8 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=


@@ 218,6 222,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=


@@ 361,6 367,8 @@ golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=


@@ 417,13 425,20 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

M main.go => main.go +16 -11
@@ 23,20 23,25 @@ import (
	"github.com/mkideal/cli"
	"log"
	"os"
	"strings"
)

// A Postgres to Sqlite exporter/converter.
// CLI Flags:
// - PG URL
// - SQLite File
// - Source Table name
type stringListDecoder struct {
	List []string
}

func (d *stringListDecoder) Decode(s string) error {
	d.List = strings.Split(s, ",")
	return nil
}

type argT struct {
	cli.Helper
	PGURL             string `cli:"*pg-url" usage:"Postgres connection string (i.e. postgres://localhost:5432/mydb)"`
	SLFile            string `cli:"*sqlite-file" usage:"Path to SQLite database file (i.e. mydatabase.db)"`
	Tablename         string `cli:"*table" usage:"Name of table to export"`
	DropTableIfExists bool   `cli:"drop-table-if-exists" usage:"DANGER: Drop target table if it already exists" default:"false"`
	PGURL             string            `cli:"*pg-url" usage:"Postgres connection string (i.e. postgres://localhost:5432/mydb)"`
	SLFile            string            `cli:"*sqlite-file" usage:"Path to SQLite database file (i.e. mydatabase.db)"`
	Tablename         string            `cli:"*table" usage:"Name of table to export"`
	IgnoreColumns     stringListDecoder `cli:"ignore-columns" usage:"comma-separated list of columns to ignore" default:""`
	DropTableIfExists bool              `cli:"drop-table-if-exists" usage:"DANGER: Drop target table if it already exists" default:"false"`
}

func (argv *argT) AutoHelp() bool {


@@ 54,7 59,7 @@ func run(ctx *cli.Context) error {
		log.Fatal(err)
	}

	schema, err := FetchSchema(argv.Tablename)
	schema, err := FetchSchema(argv.Tablename, argv.IgnoreColumns.List)
	if err != nil {
		log.Fatal(err)
	}


@@ 105,7 110,7 @@ func run(ctx *cli.Context) error {
	}()

	go func() {
		if err := LoadData(schema.Name, rowChan); err != nil {
		if err := LoadData(schema, rowChan); err != nil {
			log.Println("Error while loading data", err)
			finished <- true
		}

M pg.go => pg.go +16 -31
@@ 21,7 21,6 @@ import (
	"context"
	"fmt"
	"github.com/jackc/pgx/v4"
	"strconv"
	"strings"
)



@@ 35,8 34,9 @@ type TableSchema struct {
}

type TableColumn struct {
	Name string
	Type string
	Name    string
	Type    string
	Ignored bool
}

func ValidatePG(connStr string) error {


@@ 48,7 48,7 @@ func ValidatePG(connStr string) error {
	return nil
}

func FetchSchema(tablename string) (*TableSchema, error) {
func FetchSchema(tablename string, ignoredColumns []string) (*TableSchema, error) {
	rows, err := pgConn.Query(context.Background(),
		"SELECT column_name, data_type FROM information_schema.columns "+
			"WHERE table_name = $1 "+


@@ 73,8 73,9 @@ func FetchSchema(tablename string) (*TableSchema, error) {
			return nil, fmt.Errorf("Unable to scan columns from Postgres table: %w\n", err)
		}
		tableSchema.Cols = append(tableSchema.Cols, TableColumn{
			Name: columnName,
			Type: dataType,
			Name:    columnName,
			Type:    dataType,
			Ignored: Contains(ignoredColumns, columnName),
		})
		colCount++
	}


@@ 85,35 86,19 @@ func FetchSchema(tablename string) (*TableSchema, error) {
	return &tableSchema, nil
}

func PrintSchema(schema *TableSchema) {
	fmt.Printf("Schema of table \"%s\"\n", schema.Name)

	// Find length of widest column
	maxColLength := 0
	maxTypeLength := 0

func LoadData(schema *TableSchema, out chan []interface{}) error {
	var colListArray []string
	for _, col := range schema.Cols {
		if l := len(col.Name); maxColLength < l {
			maxColLength = l
		if col.Ignored {
			continue
		}
		if l := len(col.Type); maxTypeLength < l {
			maxTypeLength = l
		}
	}

	tmpl := "%-" + strconv.Itoa(maxColLength) + "s | %-" + strconv.Itoa(maxTypeLength) + "s\n"
	fmt.Printf(tmpl, "Column", "Type") // Header
	fmt.Printf(tmpl, strings.Repeat("-", maxColLength), strings.Repeat("-", maxTypeLength))

	for _, col := range schema.Cols {
		fmt.Printf(tmpl, col.Name, col.Type)
		colListArray = append(colListArray, fmt.Sprintf(`"%s"`, col.Name))
	}

}

func LoadData(tablename string, out chan []interface{}) error {
	sqlTmpl := "SELECT * FROM %s T"
	sqlStmt := fmt.Sprintf(sqlTmpl, tablename)
	sqlStmt := fmt.Sprintf("SELECT %s FROM %s T", strings.Join(colListArray, ", "), schema.Name)
	fmt.Println("Loading data with this statement:")
	fmt.Println(sqlStmt)
	fmt.Println()

	rows, err := pgConn.Query(context.Background(), sqlStmt)
	if err != nil {

M sqlite.go => sqlite.go +3 -0
@@ 74,6 74,9 @@ func BuildCreateTableSQL(schema *TableSchema) (string, error) {
	}

	for _, col := range schema.Cols {
		if col.Ignored {
			continue
		}
		newType, err := mapColumnType(col.Type)
		if err != nil {
			return "", fmt.Errorf("error during column type mapping: %w", err)

M ui.go => ui.go +28 -0
@@ 21,6 21,8 @@ import (
	"bufio"
	"fmt"
	"os"
	"strconv"
	"strings"
)

func AskYesNo(prompt string) bool {


@@ 39,3 41,29 @@ func AskYesNo(prompt string) bool {
		}
	}
}

func PrintSchema(schema *TableSchema) {
	fmt.Printf("Schema of table \"%s\"\n", schema.Name)

	// Find length of widest column
	maxColLength := 0
	maxTypeLength := 0

	for _, col := range schema.Cols {
		if l := len(col.Name); maxColLength < l {
			maxColLength = l
		}
		if l := len(col.Type); maxTypeLength < l {
			maxTypeLength = l
		}
	}

	tmpl := "%-" + strconv.Itoa(maxColLength) + "s | %-" + strconv.Itoa(maxTypeLength) + "s | %s\n"
	fmt.Printf(tmpl, "Column", "Type", "Ignore") // Header
	fmt.Printf(tmpl, strings.Repeat("-", maxColLength), strings.Repeat("-", maxTypeLength), strings.Repeat("-", 6))

	for _, col := range schema.Cols {
		fmt.Printf(tmpl, col.Name, col.Type, map[bool]string{true: "Yes", false: "No"}[col.Ignored])
	}
	fmt.Println()
}

A utils.go => utils.go +10 -0
@@ 0,0 1,10 @@
package main

func Contains(a []string, x string) bool {
	for _, n := range a {
		if x == n {
			return true
		}
	}
	return false
}