SF.net SVN: gar:[23750] csw/mgar/gar/v2/go
wahwah at users.sourceforge.net
wahwah at users.sourceforge.net
Sun Jun 1 23:08:15 CEST 2014
Revision: 23750
http://sourceforge.net/p/gar/code/23750
Author: wahwah
Date: 2014-06-01 21:08:13 +0000 (Sun, 01 Jun 2014)
Log Message:
-----------
Automated catalog integrations: work in progress
A basic pipeline is set up. There is no logic yet.
Modified Paths:
--------------
csw/mgar/gar/v2/go/Makefile
csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go
csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go
Added Paths:
-----------
csw/mgar/gar/v2/go/src/opencsw/mantis/
csw/mgar/gar/v2/go/src/opencsw/mantis/mantis.go
Modified: csw/mgar/gar/v2/go/Makefile
===================================================================
--- csw/mgar/gar/v2/go/Makefile 2014-06-01 20:24:40 UTC (rev 23749)
+++ csw/mgar/gar/v2/go/Makefile 2014-06-01 21:08:13 UTC (rev 23750)
@@ -8,8 +8,6 @@
BINARIES += bin/gen-catalog-index
BINARIES += bin/promote-packages
-LIBS = diskformat
-
all: bin $(BINARIES)
bin:
@@ -19,34 +17,23 @@
mkdir -p opencsw
bin/catalog-release-to-disk: src/catalog-release-to-disk/catalog-release-to-disk.o src/opencsw/diskformat/diskformat.o
- gccgo -g -o $@ $?
+ gccgo -g -o $@ $^
bin/gen-catalog-index: src/gen-catalog-index/gen-catalog-index.o src/opencsw/diskformat/diskformat.o
- gccgo -g -o $@ $?
+ gccgo -g -o $@ $^
bin/crashtest: src/crashtest/crashtest.go opencsw/diskformat.o
- gccgo -g -o $@ $?
+ gccgo -g -o $@ $^
-bin/promote-packages: src/promote-packages/promote-packages.go opencsw/diskformat.o
- gccgo -g -o $@ $?
+bin/promote-packages: src/promote-packages/promote-packages.go opencsw/diskformat.o opencsw/mantis.o
+ gccgo -g -o $@ $^
opencsw/diskformat.o: opencsw src/opencsw/diskformat/diskformat.o
- # Not portable to Linux. But we can't just use 'install' on Solaris.
ginstall -m 755 src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
-# This is a poor hack, but it gets it to compile.
-src/catalog-release-to-disk/catalog-release-to-disk.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
- gccgo -o $@ -g -c src/catalog-release-to-disk/catalog-release-to-disk.go
+opencsw/mantis.o: opencsw src/opencsw/mantis/mantis.o
+ ginstall -m 755 src/opencsw/mantis/mantis.o opencsw/mantis.o
-src/gen-catalog-index/gen-catalog-index.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
- gccgo -o $@ -g -c src/gen-catalog-index/gen-catalog-index.go
-
-src/promote-packages/promote-packages.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
- gccgo -o $@ -g -c src/promote-packages/promote-packages.go
-
-src/crashtest/crashtest.o: src/opencsw/diskformat/diskformat.o opencsw/diskformat.o
- gccgo -o $@ -g -c src/crashtest/crashtest.go
-
%.o: %.go
gccgo -o $@ -g -c $<
Modified: csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go
===================================================================
--- csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go 2014-06-01 20:24:40 UTC (rev 23749)
+++ csw/mgar/gar/v2/go/src/opencsw/diskformat/diskformat.go 2014-06-01 21:08:13 UTC (rev 23750)
@@ -92,7 +92,7 @@
// A line in the catalog file; plus the extra description field.
// As described in the manual:
-// http://www.opencsw.org/manual/for-maintainers/catalog-format.html
+// http://www.opencsw.org/manual/for-maintainers/catalog-format.html
type Package struct {
Catalogname string `json:"catalogname"`
Version string `json:"version"`
@@ -133,6 +133,30 @@
Pkgs []Package
}
+// Extra information about a package
+type PackageExtra struct {
+ Basename string `json:"basename"`
+ Bundle string `json:"bundle"`
+ Catalogname string `json:"catalogname"`
+ Category string `json:"category"`
+ Deps []string `json:"deps"`
+ I_deps []string `json:"i_deps"`
+ Inserted_by string `json:"inserted_by"` // decode?
+ Inserted_on string `json:"inserted_on"` // decode?
+ Maintainer string `json:"maintainer"`
+ Md5_sum string `json:"md5_sum"`
+ Mtime string `json:"mtime"` // decode?
+ Pkgname string `json:"pkgname"`
+ Size int `json:"size"`
+ Version string `json:"version"`
+}
+
+// I'm not sure if this is a good idea
+type CatalogExtra struct {
+ Spec CatalogSpec
+ PkgsExtra []PackageExtra
+}
+
const (
symlink = iota
hardlink
@@ -211,7 +235,7 @@
catspecs := make([]CatalogSpec, 0)
dec := json.NewDecoder(resp.Body)
if err := dec.Decode(&catspecs); err != nil {
- log.Println("Failed to decode JSON from", url)
+ log.Println("Failed to decode JSON from", url, ":", err)
return nil, err
}
@@ -253,6 +277,30 @@
return cws, nil
}
+func FetchCatalogExtra(cs CatalogSpec) (CatalogExtra, error) {
+ url := fmt.Sprintf("%s/catalogs/%s/%s/%s/timing/",
+ PkgdbUrl, cs.Catrel, cs.Arch, cs.Osrel)
+ log.Println("Making a request to", url)
+ resp, err := http.Get(url)
+ if err != nil {
+ log.Fatalln("Could not retrieve", url, ":", err)
+ return CatalogExtra{}, err
+ }
+ defer resp.Body.Close()
+
+ var pkgs []PackageExtra
+ dec := json.NewDecoder(resp.Body)
+ if err := dec.Decode(&pkgs); err != nil {
+ log.Println("Failed to decode JSON output from", url, ":", err)
+ return CatalogExtra{}, err
+ }
+
+ log.Println("Retrieved timing/extra info for", cs, "with", len(pkgs), "packages")
+
+ cws := CatalogExtra{cs, pkgs}
+ return cws, nil
+}
+
func FilterCatspecs(all_catspecs []CatalogSpec, catrel string) []CatalogSpec {
catspecs := make([]CatalogSpec, 0)
for _, catspec := range all_catspecs {
@@ -788,7 +836,7 @@
all_catspecs, err := GetCatalogSpecsFromDatabase()
if err != nil {
- log.Fatalln("Could not get the catalog spec list")
+ log.Fatalln("Could not get the catalog spec list:", err)
}
// Because of memory constraints, we're only processing 1 catalog in one run
catspecs := FilterCatspecs(all_catspecs, catrel)
Added: csw/mgar/gar/v2/go/src/opencsw/mantis/mantis.go
===================================================================
--- csw/mgar/gar/v2/go/src/opencsw/mantis/mantis.go (rev 0)
+++ csw/mgar/gar/v2/go/src/opencsw/mantis/mantis.go 2014-06-01 21:08:13 UTC (rev 23750)
@@ -0,0 +1,46 @@
+// Package mantis provides Mantis related code, allowing e.g. to look up
+// package bugs.
+package mantis
+
+import (
+ "net/http"
+ "encoding/json"
+ "log"
+
+)
+
+// Matches the data structure in http://www.opencsw.org/buglist/json
+// Maybe this should go into a separate package, like opencsw/mantis.
+type Bug struct {
+ bug_assigned_to string `json:"bug_assigned_to"`
+ bug_assigned_to_fullname string `json:"bug_assigned_to_fullname"`
+ bug_id string `json:"bug_id"`
+ bug_last_updated string `json:"bug_last_updated"`
+ bug_pkg_catalogname string `json:"bug_pkg_catalogname"`
+ bug_severity string `json:"bug_severity"`
+ bug_severity_name string `json:"bug_severity_name"`
+ bug_status string `json:"bug_status"`
+ bug_status_name string `json:"bug_status_name"`
+ bug_summary string `json:"bug_summary"`
+ maintainer_fullname string `json:"maintainer_fullname"`
+ maintainer_name string `json:"maintainer_name"`
+ maintainer_status string `json:"maintainer_status"`
+}
+
+// Fetches all bugs from Mantis.
+func FetchBugs() ([]Bug, error) {
+ url := "http://www.opencsw.org/buglist/json"
+ resp, err := http.Get(url)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ bugs := make([]Bug, 0)
+ dec := json.NewDecoder(resp.Body)
+ if err := dec.Decode(&bugs); err != nil {
+ return nil, err
+ }
+ log.Println("Fetched", len(bugs), "bugs from", url)
+ return bugs, nil
+}
Modified: csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go
===================================================================
--- csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go 2014-06-01 20:24:40 UTC (rev 23749)
+++ csw/mgar/gar/v2/go/src/promote-packages/promote-packages.go 2014-06-01 21:08:13 UTC (rev 23750)
@@ -6,10 +6,12 @@
import (
// "bufio"
+ "fmt"
"flag"
"log"
// "os"
"opencsw/diskformat"
+ "opencsw/mantis"
)
// Command line flags
@@ -36,6 +38,15 @@
toCatspec diskformat.CatalogSpec
}
+type CatalogWithSpecTransition struct {
+ fromCat diskformat.CatalogExtra
+ toCat diskformat.CatalogExtra
+}
+
+type IntegrationResult struct {
+ noidea string
+}
+
func groupByOsrelAndArch(cs []diskformat.CatalogSpec) map[diskformat.CatalogSpec]diskformat.CatalogSpec {
fromIndexed := make(map[diskformat.CatalogSpec]diskformat.CatalogSpec)
for _, f := range cs {
@@ -61,18 +72,18 @@
return transitions
}
-func fetchTwo(transition CatalogSpecTransition) (f, t diskformat.CatalogWithSpec) {
- chf := make(chan diskformat.CatalogWithSpec)
- go func(ch chan diskformat.CatalogWithSpec) {
- fromCat, err := diskformat.GetCatalogWithSpec(transition.fromCatspec)
+func fetchTwo(transition CatalogSpecTransition) (f, t diskformat.CatalogExtra) {
+ chf := make(chan diskformat.CatalogExtra)
+ go func(ch chan diskformat.CatalogExtra) {
+ fromCat, err := diskformat.FetchCatalogExtra(transition.fromCatspec)
if err != nil {
log.Fatalln("Could not fetch", fromCat, "error:", err)
}
ch <- fromCat
}(chf)
- cht := make(chan diskformat.CatalogWithSpec)
- go func(ch chan diskformat.CatalogWithSpec) {
- toCat, err := diskformat.GetCatalogWithSpec(transition.toCatspec)
+ cht := make(chan diskformat.CatalogExtra)
+ go func(ch chan diskformat.CatalogExtra) {
+ toCat, err := diskformat.FetchCatalogExtra(transition.toCatspec)
if err != nil {
log.Fatalln("Could not fetch", toCat, "error:", err)
}
@@ -92,20 +103,93 @@
// Package addition and removal times are not taken from the catalog, but
// from the times when we saw packages appear and/or disappear.
// How to test these rules? What should be the output of this function?
+ log.Println("Example package:", fromCat.PkgsExtra[0])
+ log.Println("Example package:", toCat.PkgsExtra[0])
+
+ // Mantis:
+ // http://www.opencsw.org/buglist/json
}
-func main() {
- flag.Parse()
- log.Println("Program start")
+func transitions() []CatalogSpecTransition {
all_catspecs, err := diskformat.GetCatalogSpecsFromDatabase()
if err != nil {
log.Fatalln("Could not get the catalog spec list")
}
from_catspecs := diskformat.FilterCatspecs(all_catspecs, from_catrel_flag)
to_catspecs := diskformat.FilterCatspecs(all_catspecs, to_catrel_flag)
- transitions := matchCatspecs(from_catspecs, to_catspecs)
- for _, transition := range transitions {
- Integrate(transition)
+ return matchCatspecs(from_catspecs, to_catspecs)
+}
+
+// The idea of creating a pipeline in this fashion is that a function call
+// generates a channel and closes it after all data are written to it.
+// The closure inside this function looks weird, but its scope is the same as
+// the scope of the function, so it quickly becomes natural to read, and keeps
+// related pieces of code together.
+
+func pipeStage1() <-chan CatalogSpecTransition {
+ out := make(chan CatalogSpecTransition)
+ go func() {
+ for _, t := range transitions() {
+ out <- t
+ }
+ close(out)
+ }()
+ return out
+}
+
+func pipeStage2(in <-chan CatalogSpecTransition) <-chan CatalogWithSpecTransition {
+ out := make(chan CatalogWithSpecTransition)
+ go func() {
+ for t := range in {
+ fromCat, toCat := fetchTwo(t)
+ out <-CatalogWithSpecTransition{fromCat, toCat}
+ }
+ close(out)
+ }()
+ return out
+}
+
+// Continue from here: write the 3rd stage which just prints the results.
+func pipeStage3(in <-chan CatalogWithSpecTransition) <-chan IntegrationResult {
+ out := make(chan IntegrationResult)
+ go func() {
+ for t := range in {
+ msg := fmt.Sprintf("Processing our fetched data: %+v -> %+v",
+ t.fromCat.Spec, t.toCat.Spec)
+ out <-IntegrationResult{msg}
+ }
+ close(out)
+ }()
+ return out
+}
+
+func mantisChan() <-chan []mantis.Bug {
+ mch := make(chan []mantis.Bug)
+ go func(ch chan []mantis.Bug) {
+ log.Println("Fetching bugs from mantis")
+ bugs, err := mantis.FetchBugs()
+ if err != nil {
+ log.Fatalln("Fetching bugs failed.")
+ }
+ log.Println("Fetched", len(bugs), "bugs from Mantis")
+ ch <-bugs
+ close(ch)
+ }(mch)
+ return mch
+}
+
+func main() {
+ flag.Parse()
+ log.Println("Program start")
+
+ mch := mantisChan()
+ tch := pipeStage1()
+ cch := pipeStage2(tch)
+ rch := pipeStage3(cch)
+
+ for r := range rch {
+ log.Println("Result:", r)
}
+ <-mch
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
More information about the devel
mailing list