SF.net SVN: gar:[23724] csw/mgar/gar/v2/go
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Thu May 29 00:39:13 CEST 2014
Revision: 23724
http://sourceforge.net/p/gar/code/23724
Author: wahwah
Date: 2014-05-28 22:39:12 +0000 (Wed, 28 May 2014)
Log Message:
-----------
Automatic catalog integrations: Work in progress
Modified Paths:
--------------
csw/mgar/gar/v2/go/Makefile
csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go
Added Paths:
-----------
csw/mgar/gar/v2/go/src/promote-packages/
csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go
Modified: csw/mgar/gar/v2/go/Makefile
===================================================================
--- csw/mgar/gar/v2/go/Makefile 2014-05-28 22:39:04 UTC (rev 23723)
+++ csw/mgar/gar/v2/go/Makefile 2014-05-28 22:39:12 UTC (rev 23724)
@@ -3,30 +3,53 @@
# 1. gmake
# 2. scp to the web host to replace the old binary
-all: bin bin/catalog-release-to-disk bin/gen-catalog-index
+BINARIES += bin/catalog-release-to-disk
+BINARIES += bin/crashtest
+BINARIES += bin/gen-catalog-index
+BINARIES += bin/promote-packages
+LIBS = diskformat
+
+all: bin $(BINARIES)
+
bin:
mkdir -p bin
+opencsw:
+ mkdir -p opencsw
+
bin/catalog-release-to-disk: src/catalog-release-to-disk/catalog-release-to-disk.o src/opencsw/diskformat/diskformat.o
gccgo -g -o $@ $?
bin/gen-catalog-index: src/gen-catalog-index/gen-catalog-index.o src/opencsw/diskformat/diskformat.o
gccgo -g -o $@ $?
+bin/crashtest: src/crashtest/crashtest.go opencsw/diskformat.o
+ gccgo -g -o $@ $?
+
+bin/promote-packages: src/promote-packages/promote-packages.go opencsw/diskformat.o
+ gccgo -g -o $@ $?
+
+opencsw/diskformat.o: opencsw src/opencsw/diskformat/diskformat.o
+ # Not portable to Linux. But we can't just use 'install' on Solaris.
+ ginstall -m 755 src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
+
# This is a poor hack, but it gets it to compile.
-src/catalog-release-to-disk/catalog-release-to-disk.o: src/opencsw/diskformat/diskformat.o
- mkdir -p opencsw
- ln -sf ../src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
+src/catalog-release-to-disk/catalog-release-to-disk.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
gccgo -o $@ -g -c src/catalog-release-to-disk/catalog-release-to-disk.go
-src/gen-catalog-index/gen-catalog-index.o: src/opencsw/diskformat/diskformat.o
- mkdir -p opencsw
- ln -sf ../src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
+src/gen-catalog-index/gen-catalog-index.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
gccgo -o $@ -g -c src/gen-catalog-index/gen-catalog-index.go
+src/promote-packages/promote-packages.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
+ gccgo -o $@ -g -c src/promote-packages/promote-packages.go
+
+src/crashtest/crashtest.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
+ gccgo -o $@ -g -c src/crashtest/crashtest.go
+
%.o: %.go
gccgo -o $@ -g -c $<
clean:
- rm -f src/opencsw/diskformat/diskformat.o bin/catalog-release-to-disk bin/gen-catalog-index
+ find . -name '*.o' -exec rm -f {} \;
+ rm -f $(BINARIES)
Modified: csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go
===================================================================
--- csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go 2014-05-28 22:39:04 UTC (rev 23723)
+++ csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go 2014-05-28 22:39:12 UTC (rev 23724)
@@ -27,6 +27,7 @@
// Keeping PkgdbUrl as a package global variable is probably not the best idea,
// but let's not refactor without a good plan.
var PkgdbUrl string
+var ReleasesUrl string
// 3 strings that define a specific catalog, e.g. "unstable sparc 5.10"
type CatalogSpec struct {
@@ -188,6 +189,17 @@
return nil
}
+// Returns True when the two catalog specs match in arch and osrel.
+func (s CatalogSpec) Matches(o CatalogSpec) bool {
+ if s.Arch == o.Arch && s.Osrel == o.Osrel {
+ if s.Catrel == o.Catrel {
+ log.Println("We're matching the same catspec against itself: ", s)
+ }
+ return true
+ }
+ return false
+}
+
func GetCatalogSpecsFromDatabase() ([]CatalogSpec, error) {
url := fmt.Sprintf("%s/catalogs/", PkgdbUrl)
resp, err := http.Get(url)
@@ -241,7 +253,7 @@
return cws, nil
}
-func filterCatspecs(all_catspecs []CatalogSpec, catrel string) []CatalogSpec {
+func FilterCatspecs(all_catspecs []CatalogSpec, catrel string) []CatalogSpec {
catspecs := make([]CatalogSpec, 0)
for _, catspec := range all_catspecs {
if catspec.Catrel == catrel {
@@ -776,10 +788,10 @@
all_catspecs, err := GetCatalogSpecsFromDatabase()
if err != nil {
- log.Panicln("Could not get the catalog spec list")
+ log.Fatalln("Could not get the catalog spec list")
}
// Because of memory constraints, we're only processing 1 catalog in one run
- catspecs := filterCatspecs(all_catspecs, catrel)
+ catspecs := FilterCatspecs(all_catspecs, catrel)
// The plan:
// 1. build a data structure representing all the hardlinks and symlinks
Added: csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go
===================================================================
--- csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go (rev 0)
+++ csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go 2014-05-28 22:39:12 UTC (rev 23724)
@@ -0,0 +1,111 @@
+// Command promote-packages analyzes the state of the catalogs, and promotes
+// packages from one catalog (unstable) to another one, based on a set of
+// rules.
+
+package main
+
+import (
+ // "bufio"
+ "flag"
+ "log"
+ // "os"
+ "opencsw/diskformat"
+)
+
+// Command line flags
+var from_catrel_flag string
+var to_catrel_flag string
+
+func init() {
+ flag.StringVar(&from_catrel_flag, "from-catrel", "unstable",
+ "Actually, only unstable makes sense here.")
+ flag.StringVar(&to_catrel_flag, "to-catrel", "bratislava",
+ "The testing release.")
+ flag.BoolVar(&diskformat.DryRun, "dry-run", true,
+ "Don't make any changes.")
+ flag.StringVar(&diskformat.PkgdbUrl, "pkgdb-url",
+ "http://buildfarm.opencsw.org/pkgdb/rest",
+ "Web address of the pkgdb app.")
+ flag.StringVar(&diskformat.ReleasesUrl, "releases-url",
+ "http://buildfarm.opencsw.org/releases",
+ "Web address of the releases app.")
+}
+
+type CatalogSpecTransition struct {
+ fromCatspec diskformat.CatalogSpec
+ toCatspec diskformat.CatalogSpec
+}
+
+func groupByOsrelAndArch(cs []diskformat.CatalogSpec) map[diskformat.CatalogSpec]diskformat.CatalogSpec {
+ fromIndexed := make(map[diskformat.CatalogSpec]diskformat.CatalogSpec)
+ for _, f := range cs {
+ i := diskformat.CatalogSpec{"fake", f.Arch, f.Osrel}
+ fromIndexed[i] = f
+ }
+ return fromIndexed
+}
+
+func matchCatspecs(fromCats, toCats []diskformat.CatalogSpec) ([]CatalogSpecTransition) {
+ transitions := make([]CatalogSpecTransition, 0)
+ // We need to match the catspecs according to osrel and arch.
+ fromIndexed := groupByOsrelAndArch(fromCats)
+ toIndexed := groupByOsrelAndArch(toCats)
+ for k, f := range fromIndexed {
+ if t, ok := toIndexed[k]; ok {
+ transitions = append(transitions, CatalogSpecTransition{f, t})
+ } else {
+ log.Println("Did not find a match for", f, "something might be wrong",
+ "with the data / the database.")
+ }
+ }
+ return transitions
+}
+
+func fetchTwo(transition CatalogSpecTransition) (f, t diskformat.CatalogWithSpec) {
+ chf := make(chan diskformat.CatalogWithSpec)
+ go func(ch chan diskformat.CatalogWithSpec) {
+ fromCat, err := diskformat.GetCatalogWithSpec(transition.fromCatspec)
+ if err != nil {
+ log.Fatalln("Could not fetch", fromCat, "error:", err)
+ }
+ ch <- fromCat
+ }(chf)
+ cht := make(chan diskformat.CatalogWithSpec)
+ go func(ch chan diskformat.CatalogWithSpec) {
+ toCat, err := diskformat.GetCatalogWithSpec(transition.toCatspec)
+ if err != nil {
+ log.Fatalln("Could not fetch", toCat, "error:", err)
+ }
+ ch <- toCat
+ }(cht)
+ return <-chf, <-cht
+}
+
+func Integrate(transition CatalogSpecTransition) {
+ // We have the catalogs, now we need to generate commands to integrate the
+ // catalogs. One of the questions is how do we map replacements from one
+ // catalog into the other; and how do we group these catalogs.
+ fromCat, toCat := fetchTwo(transition)
+ log.Println("Integration from", fromCat.Spec, "to", toCat.Spec)
+ // Integrations are done by bundle; a group of packages. (or maybe not?)
+ // Considering deletions as well as additions of packages.
+ // Package addition and removal times are not taken from the catalog, but
+ // from the times when we saw packages appear and/or disappear.
+ // How to test these rules? What should be the output of this function?
+}
+
+func main() {
+ flag.Parse()
+ log.Println("Program start")
+
+ all_catspecs, err := diskformat.GetCatalogSpecsFromDatabase()
+ if err != nil {
+ log.Fatalln("Could not get the catalog spec list")
+ }
+ from_catspecs := diskformat.FilterCatspecs(all_catspecs, from_catrel_flag)
+ to_catspecs := diskformat.FilterCatspecs(all_catspecs, to_catrel_flag)
+ transitions := matchCatspecs(from_catspecs, to_catspecs)
+ for _, transition := range transitions {
+ Integrate(transition)
+ }
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list