Initial
This commit is contained in:
Alexander NeonXP Kiryukhin
commit
1f5a78cb8d
6 changed files with 334 additions and 0 deletions
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
.idea
|
||||
osm2mgo
|
||||
22
README.md
Normal file
22
README.md
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# OpenStreetMaps to Mongo
|
||||
|
||||
Simple loader from osm dump file to mongodb. Based on https://github.com/paulmach/osm package.
|
||||
|
||||
## Build
|
||||
|
||||
`go build -o osm2go`
|
||||
|
||||
## Usage
|
||||
|
||||
`./osm2go -osmfile PATH_TO_OSM_FILE [-dbconnection mongodb://localhost:27017] [-dbname osm]`
|
||||
|
||||
* `osmfile` required, path to *.osm or *.osm.pbf file
|
||||
* `dbconnection` optional, mongodb connection string (default: `mongodb://localhost:27017`)
|
||||
* `dbname` optional, mongodb database name (default: `osm`)
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
# ./osm2mgo -osmfile ~/Downloads/RU.pbf
|
||||
Nodes: 1294069 Ways: 0 Relations: 0
|
||||
```
|
||||
20
go.mod
Normal file
20
go.mod
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
module osm2mongo
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/gogo/protobuf v1.2.1 // indirect
|
||||
github.com/golang/snappy v0.0.1 // indirect
|
||||
github.com/google/go-cmp v0.3.0 // indirect
|
||||
github.com/paulmach/orb v0.1.3
|
||||
github.com/paulmach/osm v0.0.1
|
||||
github.com/stretchr/testify v1.3.0 // indirect
|
||||
github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 // indirect
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
|
||||
github.com/xdg/stringprep v1.0.0 // indirect
|
||||
go.mongodb.org/mongo-driver v1.0.2
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
)
|
||||
42
go.sum
Normal file
42
go.sum
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
|
||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM=
|
||||
github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
|
||||
github.com/paulmach/osm v0.0.1 h1:TxU/uZnJ+ssntblY6kSieOP4tQv31VW8wfyf4L3BeKs=
|
||||
github.com/paulmach/osm v0.0.1/go.mod h1:d6Ez0X1es6TaiYBxNmn1WbutRRQleZdvpoqeWdH/5kM=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g=
|
||||
github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
go.mongodb.org/mongo-driver v1.0.2 h1:RwjK1tKt7VPqQh3tsjiEqKJg75GNhP/loch+PwRc4ig=
|
||||
go.mongodb.org/mongo-driver v1.0.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
|
||||
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
191
main.go
Normal file
191
main.go
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/paulmach/osm"
|
||||
"github.com/paulmach/osm/osmpbf"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
"go.mongodb.org/mongo-driver/x/bsonx"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dbconnection := flag.String("dbconnection", "mongodb://localhost:27017", "Mongo database name")
|
||||
dbname := flag.String("dbname", "osm", "Mongo database name")
|
||||
osmfile := flag.String("osmfile", "", "OSM file")
|
||||
flag.Parse()
|
||||
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(*dbconnection))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer client.Disconnect(context.Background())
|
||||
db := client.Database(*dbname)
|
||||
if err := read(db, *osmfile); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func read(db *mongo.Database, file string) error {
|
||||
nodes := db.Collection("nodes")
|
||||
_, _ = nodes.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetUnique(true).SetSparse(true),
|
||||
},
|
||||
)
|
||||
_, _ = nodes.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"coords", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetSphereVersion(2).SetSparse(true),
|
||||
},
|
||||
)
|
||||
|
||||
ways := db.Collection("ways")
|
||||
_, _ = ways.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetUnique(true).SetSparse(true),
|
||||
},
|
||||
)
|
||||
_, _ = ways.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"nodes", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetSparse(true),
|
||||
},
|
||||
)
|
||||
|
||||
relations := db.Collection("relations")
|
||||
_, _ = nodes.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"osm_id", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetUnique(true).SetSparse(true),
|
||||
},
|
||||
)
|
||||
_, _ = nodes.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"members.ref", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetUnique(true).SetSparse(true),
|
||||
},
|
||||
)
|
||||
_, _ = nodes.Indexes().CreateOne(
|
||||
context.Background(),
|
||||
mongo.IndexModel{
|
||||
Keys: bsonx.Doc{{"members.coords", bsonx.Int32(1)}},
|
||||
Options: options.Index().SetSphereVersion(2).SetSparse(true),
|
||||
},
|
||||
)
|
||||
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
opts := (new(options.ReplaceOptions)).SetUpsert(true)
|
||||
nc := 0
|
||||
wc := 0
|
||||
rc := 0
|
||||
|
||||
scanner := osmpbf.New(context.Background(), f, 3)
|
||||
defer scanner.Close()
|
||||
|
||||
for scanner.Scan() {
|
||||
o := scanner.Object()
|
||||
switch o := o.(type) {
|
||||
case *osm.Way:
|
||||
nodes := make([]int64, 0, len(o.Nodes))
|
||||
for _, v := range o.Nodes {
|
||||
nodes = append(nodes, int64(v.ID))
|
||||
}
|
||||
w := Way{
|
||||
OsmID: int64(o.ID),
|
||||
Tags: convertTags(o.Tags),
|
||||
Nodes: nodes,
|
||||
Timestamp: o.Timestamp,
|
||||
Version: o.Version,
|
||||
Visible: o.Visible,
|
||||
}
|
||||
if _, err = ways.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, w, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
wc++
|
||||
case *osm.Node:
|
||||
n := Node{
|
||||
OsmID: int64(o.ID),
|
||||
Location: Coords{
|
||||
Type: "Point",
|
||||
Coordinates: []float64{
|
||||
o.Lon,
|
||||
o.Lat,
|
||||
}},
|
||||
Tags: convertTags(o.Tags),
|
||||
Version: o.Version,
|
||||
Timestamp: o.Timestamp,
|
||||
Visible: o.Visible,
|
||||
}
|
||||
if _, err = nodes.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, n, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
nc++
|
||||
case *osm.Relation:
|
||||
members := make([]Member, len(o.Members))
|
||||
for _, v := range o.Members {
|
||||
members = append(members, Member{
|
||||
Type: v.Type,
|
||||
Version: v.Version,
|
||||
Orientation: v.Orientation,
|
||||
Ref: v.Ref,
|
||||
Role: v.Role,
|
||||
Location: Coords{
|
||||
Type: "Point",
|
||||
Coordinates: []float64{
|
||||
v.Lon,
|
||||
v.Lat,
|
||||
}},
|
||||
})
|
||||
}
|
||||
r := Relation{
|
||||
OsmID: int64(o.ID),
|
||||
Tags: convertTags(o.Tags),
|
||||
Version: o.Version,
|
||||
Timestamp: o.Timestamp,
|
||||
Visible: o.Visible,
|
||||
Members: members,
|
||||
}
|
||||
if _, err = relations.ReplaceOne(context.Background(), bson.M{"osm_id": int64(o.ID)}, r, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
rc++
|
||||
}
|
||||
fmt.Printf("\rNodes: %d Ways: %d Relations: %d", nc, wc, rc)
|
||||
}
|
||||
|
||||
scanErr := scanner.Err()
|
||||
if scanErr != nil {
|
||||
return scanErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func convertTags(tags osm.Tags) map[string]string {
|
||||
result := make(map[string]string, len(tags))
|
||||
for _, t := range tags {
|
||||
result[t.Key] = t.Value
|
||||
}
|
||||
return result
|
||||
}
|
||||
57
models.go
Normal file
57
models.go
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/paulmach/orb"
|
||||
"github.com/paulmach/osm"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
type Coords struct {
|
||||
Type string `bson:"type"`
|
||||
Coordinates []float64 `bson:"coordinates"`
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
ID primitive.ObjectID `bson:"_id,omitempty"`
|
||||
OsmID int64 `bson:"osm_id"`
|
||||
Visible bool `bson:"visible"`
|
||||
Version int `bson:"version,omitempty"`
|
||||
Timestamp time.Time `bson:"timestamp"`
|
||||
Tags map[string]string `bson:"tags,omitempty"`
|
||||
Location Coords `bson:"location"`
|
||||
}
|
||||
|
||||
type Way struct {
|
||||
ID primitive.ObjectID `bson:"_id,omitempty"`
|
||||
OsmID int64 `bson:"osm_id"`
|
||||
Visible bool `bson:"visible"`
|
||||
Version int `bson:"version"`
|
||||
Timestamp time.Time `bson:"timestamp"`
|
||||
Nodes []int64 `bson:"nodes"`
|
||||
Tags map[string]string `bson:"tags"`
|
||||
}
|
||||
|
||||
type Relation struct {
|
||||
ID primitive.ObjectID `bson:"_id,omitempty"`
|
||||
OsmID int64 `bson:"osm_id"`
|
||||
Visible bool `bson:"visible"`
|
||||
Version int `bson:"version"`
|
||||
Timestamp time.Time `bson:"timestamp"`
|
||||
Members []Member `bson:"members"`
|
||||
Tags map[string]string `bson:"tags"`
|
||||
}
|
||||
|
||||
type Member struct {
|
||||
Type osm.Type `bson:"type"`
|
||||
Ref int64 `bson:"ref"`
|
||||
Role string `bson:"role"`
|
||||
|
||||
Version int
|
||||
Location Coords `bson:"location"`
|
||||
|
||||
// Orientation is the direction of the way around a ring of a multipolygon.
|
||||
// Only valid for multipolygon or boundary relations.
|
||||
Orientation orb.Orientation `bson:"orienation,omitempty"`
|
||||
}
|
||||
Loading…
Reference in a new issue