WIP: migrations

This commit is contained in:
Jan Dittberner 2020-12-15 20:15:40 +01:00
parent b57c01b3c4
commit e67dc820cf

View file

@ -5,12 +5,17 @@ import (
"crypto/rand" "crypto/rand"
"crypto/sha1" "crypto/sha1"
"database/sql" "database/sql"
"errors" "encoding/csv"
"fmt" "fmt"
"io"
"net/http"
"strings"
"time" "time"
"github.com/pressly/goose" "github.com/pressly/goose"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/charmap"
) )
func init() { func init() {
@ -19,14 +24,28 @@ func init() {
func Up20201214193523(tx *sql.Tx) error { func Up20201214193523(tx *sql.Tx) error {
// This code is executed when the migration is applied. // This code is executed when the migration is applied.
var row *sql.Row var (
data int
countryId, regionId, locationId int64
row *sql.Row
result sql.Result
err error
)
row = tx.QueryRow("SELECT COUNT(*) FROM users WHERE admin=1") row = tx.QueryRow("SELECT COUNT(*) FROM users WHERE admin=1")
var data int if err = row.Scan(&data); err != nil {
if err := row.Scan(&data); err != nil {
return err return err
} }
log.Infof("%d admins found\n", data) log.Infof("%d admins found\n", data)
var countryFipsCodeMap map[string]int64
if countryFipsCodeMap, err = updateCountries(tx); err != nil {
return err
}
if err = updateRegions(tx, &countryFipsCodeMap); err != nil {
return err
}
if data == 0 { if data == 0 {
location, err := time.LoadLocation("Europe/Berlin") location, err := time.LoadLocation("Europe/Berlin")
if err != nil { if err != nil {
@ -40,14 +59,16 @@ func Up20201214193523(tx *sql.Tx) error {
if err != nil { if err != nil {
return err return err
} }
row = tx.QueryRow("SELECT id FROM countries WHERE name='Germany'")
var ccid int if countryId, err = getCountryId(tx, "Germany"); err != nil {
if err := row.Scan(&ccid); err != nil { return err
}
if regionId, err = getRegionId(tx, "Sachsen", countryId); err != nil {
return err
}
if locationId, err = getLocationId(tx, "Dresden", countryId, regionId); err != nil {
return err return err
} }
regid := 0
locid := 0
random64Bytes := make([]byte, 64) random64Bytes := make([]byte, 64)
_, err = rand.Read(random64Bytes) _, err = rand.Read(random64Bytes)
@ -55,20 +76,32 @@ func Up20201214193523(tx *sql.Tx) error {
return err return err
} }
result, err := tx.Exec(`INSERT INTO users (email, password, fname, mname, result, err = tx.Exec(`INSERT INTO users (email, password, fname, mname,
lname, suffix, dob, verified, ccid, lname, suffix, dob, verified, ccid,
regid, locid, listme, codesign, 1024bit, contactinfo, admin, orgadmin, regid, locid, listme, codesign, 1024bit, contactinfo, admin, orgadmin,
ttpadmin, adadmin, board, tverify, locadmin, language, ttpadmin, adadmin, board, tverify, locadmin, language,
Q1, Q2, Q3, Q4, Q5, Q1, Q2, Q3, Q4, Q5,
A1, A2, A3, A4, A5, A1, A2, A3, A4, A5,
created, modified, deleted, locked, uniqueID, otphash, created, modified, locked, uniqueID,
otppin, assurer, assurer_blocked, lastLoginAttempt) otphash, otppin, assurer, assurer_blocked, lastLoginAttempt)
VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, 0, 1, 0, ?, 1, 0, 0, 0, 0, 0, 0, ?, '', '', '', '', '', '', '', '', '', '', ?, VALUES (?, ?, ?, '', ?, '', ?, 0,
?, NULL, 0, SHA1(CONCAT(NOW(), ?)), '', 0, 0, 0, NULL)`, ?, ?, ?, 0, 1, 0, ?,
1, 0, 0, 0, 0, 0, 0, ?,
'', '', '', '', '', '', '', '', '', '',
CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0,
SHA1(CONCAT(NOW(), ?)),
'', 0, 0, 0, NULL)`,
"jandd@cacert.org", "jandd@cacert.org",
fmt.Sprintf("%x", sha1.Sum([]byte("abcdefghijklmn")), fmt.Sprintf("%x", sha1.Sum([]byte("abcdefghijklmn"))),
"Jan", "", "Dittberner", "", dob, ccid, regid, locid, "Somewhere over the rainbow", "Jan",
time.Now(), time.Now(), fmt.Sprintf("%x", md5.Sum(random64Bytes)))) "Dittberner",
dob,
countryId,
regionId,
locationId,
"Somewhere over the rainbow",
"de_DE",
fmt.Sprintf("%x", md5.Sum(random64Bytes)))
if err != nil { if err != nil {
return err return err
} }
@ -78,8 +111,197 @@ VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, 0, 1, 0, ?, 1, 0, 0, 0, 0, 0, 0, ?, '',
} }
log.Infof("new user id is %d", lastId) log.Infof("new user id is %d", lastId)
} }
return nil
}
return errors.New("TODO: implement") func updateRegions(tx *sql.Tx, codeMap *map[string]int64) error {
client := &http.Client{}
var (
err error
request *http.Request
response *http.Response
csvReader *csv.Reader
)
request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/fips-10-4/master/data/data.csv", nil)
if err != nil {
return err
}
response, err = client.Do(request)
if err != nil {
return err
}
if response.StatusCode != 200 {
return fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status)
}
csvReader = csv.NewReader(response.Body)
headings, err := csvReader.Read()
log.Infof("CSV headings %s", strings.Join(headings, ","))
for {
record, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
regionCode := record[0]
regionDivision := record[1]
regionName := record[2]
fipsCode := regionCode[:2]
log.Infof("read %s %s %s", regionCode, regionName, fipsCode)
var countryId int64
var exists bool
if countryId, exists = (*codeMap)[fipsCode]; exists {
log.Infof("country id %d", countryId)
} else if regionDivision == "country" {
countryId, err = getCountryId(tx, regionName)
(*codeMap)[fipsCode] = countryId
} else {
return fmt.Errorf("could not find country for %s %s", fipsCode, regionName)
}
_, err = getRegionId(tx, regionName, countryId)
if err != nil {
return err
}
}
return nil
}
func updateCountries(tx *sql.Tx) (map[string]int64, error) {
client := &http.Client{}
var (
err error
request *http.Request
response *http.Response
csvReader *csv.Reader
)
request, err = http.NewRequest("GET", "https://raw.githubusercontent.com/datasets/country-codes/master/data/country-codes.csv", nil)
if err != nil {
return nil, err
}
response, err = client.Do(request)
if err != nil {
return nil, err
}
if response.StatusCode != 200 {
return nil, fmt.Errorf("got unexpected HTTP status %d %s", response.StatusCode, response.Status)
}
csvReader = csv.NewReader(response.Body)
headings, err := csvReader.Read()
log.Infof("CSV headings %s", strings.Join(headings, ","))
countryFipsMapping := make(map[string]int64, 0)
var count int64 = 0
for {
record, err := csvReader.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
name := strings.TrimSpace(record[54])
if len(name) > 0 {
countryId, err := getCountryId(tx, name)
if err != nil {
return nil, err
}
countryFipsMapping[record[7]] = countryId
count++
}
}
log.Infof("read %d countries", count)
return countryFipsMapping, nil
}
func getLocationId(tx *sql.Tx, name string, countryId, regionId int64) (int64, error) {
var (
row *sql.Row
result sql.Result
locationId int64
err error
)
row = tx.QueryRow("SELECT id FROM locations WHERE name=? AND ccid=? AND regid=?", name, countryId, regionId)
if err := row.Scan(&locationId); err != nil {
if err != sql.ErrNoRows {
return 0, err
}
} else {
return locationId, nil
}
result, err = tx.Exec("INSERT INTO locations (regid, ccid, name, acount) VALUES (?, ?, ?, ?)", regionId, countryId, name, 0)
if err != nil {
return 0, err
}
locationId, err = result.LastInsertId()
if err != nil {
return 0, err
}
return locationId, nil
}
func getRegionId(tx *sql.Tx, name string, countryId int64) (int64, error) {
var (
row *sql.Row
result sql.Result
regionId int64
err error
)
encoder := charmap.ISO8859_1.NewEncoder()
cutDownName, err := encoding.HTMLEscapeUnsupported(encoder).String(name)
if err != nil {
return 0, err
}
if len(cutDownName) > 50 {
cutDownName = cutDownName[:50]
}
row = tx.QueryRow("SELECT id FROM regions WHERE name=? AND ccid=?", cutDownName, countryId)
if err := row.Scan(&regionId); err != nil {
if err != sql.ErrNoRows {
return 0, err
}
} else {
return regionId, nil
}
result, err = tx.Exec("INSERT INTO regions (ccid, name, acount) VALUES (?, ?, ?)", countryId, cutDownName, 0)
if err != nil {
return 0, err
}
regionId, err = result.LastInsertId()
if err != nil {
return 0, err
}
return regionId, nil
}
func getCountryId(tx *sql.Tx, name string) (int64, error) {
var (
row *sql.Row
result sql.Result
countryId int64
err error
)
row = tx.QueryRow("SELECT id FROM countries WHERE name=?", name)
if err := row.Scan(&countryId); err != nil {
if err != sql.ErrNoRows {
return 0, err
}
} else {
return countryId, nil
}
result, err = tx.Exec("INSERT INTO countries (countries.name, countries.acount) VALUES (?, ?)", name, 0)
if err != nil {
return 0, err
}
countryId, err = result.LastInsertId()
if err != nil {
return 0, err
}
return countryId, nil
} }
func Down20201214193523(tx *sql.Tx) error { func Down20201214193523(tx *sql.Tx) error {