commit - f00d51cf8cc1331170bf6c84bfa23bbe7e8f0e98
commit + de59b21200a5aa169b2dc799bca29b8f75c8e231
blob - d606a708d25244cbad788ab9b159bb707cc35ea1 (mode 644)
blob + /dev/null
--- examples/couchserver/couchserver.go
+++ /dev/null
-package main
-
-import (
- "bytes"
- "encoding/base64"
- "encoding/json"
- "flag"
- "fmt"
- "io"
- "log"
- "log/syslog"
- "net"
- "net/textproto"
- "net/url"
- "strconv"
- "strings"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/dustin/go-nntp"
- "github.com/dustin/go-nntp/server"
-
- "github.com/dustin/go-couch"
-)
-
-var groupCacheTimeout = flag.Int("groupTimeout", 300,
- "Time (in seconds), group cache is valid")
-var optimisticPost = flag.Bool("optimistic", false,
- "Optimistically return success on store before storing")
-var useSyslog = flag.Bool("syslog", false,
- "Log to syslog")
-
-type groupRow struct {
- Group string `json:"key"`
- Value []interface{} `json:"value"`
-}
-
-type groupResults struct {
- Rows []groupRow
-}
-
-type attachment struct {
- Type string `json:"content-type"`
- Data []byte `json:"data"`
-}
-
-func removeSpace(r rune) rune {
- if r == ' ' || r == '\n' || r == '\r' {
- return -1
- }
- return r
-}
-
-func (a *attachment) MarshalJSON() ([]byte, error) {
- m := map[string]string{
- "content_type": a.Type,
- "data": strings.Map(removeSpace, base64.StdEncoding.EncodeToString(a.Data)),
- }
- return json.Marshal(m)
-}
-
-type article struct {
- MsgID string `json:"_id"`
- DocType string `json:"type"`
- Headers map[string][]string `json:"headers"`
- Bytes int `json:"bytes"`
- Lines int `json:"lines"`
- Nums map[string]int64 `json:"nums"`
- Attachments map[string]*attachment `json:"_attachments"`
- Added time.Time `json:"added"`
-}
-
-type articleResults struct {
- Rows []struct {
- Key []interface{} `json:"key"`
- Article article `json:"doc"`
- }
-}
-
-type couchBackend struct {
- db *couch.Database
- groups map[string]*nntp.Group
- grouplock sync.Mutex
-}
-
-func (cb *couchBackend) clearGroups() {
- cb.grouplock.Lock()
- defer cb.grouplock.Unlock()
-
- log.Printf("Dumping group cache")
- cb.groups = nil
-}
-
-func (cb *couchBackend) fetchGroups() error {
- cb.grouplock.Lock()
- defer cb.grouplock.Unlock()
-
- if cb.groups != nil {
- return nil
- }
-
- log.Printf("Filling group cache")
-
- results := groupResults{}
- err := cb.db.Query("_design/groups/_view/active", map[string]interface{}{
- "group": true,
- }, &results)
- if err != nil {
- return err
- }
- cb.groups = make(map[string]*nntp.Group)
- for _, gr := range results.Rows {
- if gr.Value[0].(string) != "" {
- group := nntp.Group{
- Name: gr.Group,
- Description: gr.Value[0].(string),
- Count: int64(gr.Value[1].(float64)),
- Low: int64(gr.Value[2].(float64)),
- High: int64(gr.Value[3].(float64)),
- Posting: nntp.PostingPermitted,
- }
- cb.groups[group.Name] = &group
- }
- }
-
- go func() {
- time.Sleep(time.Duration(*groupCacheTimeout) * time.Second)
- cb.clearGroups()
- }()
-
- return nil
-}
-
-func (cb *couchBackend) ListGroups(max int) ([]*nntp.Group, error) {
- if cb.groups == nil {
- if err := cb.fetchGroups(); err != nil {
- return nil, err
- }
- }
- rv := make([]*nntp.Group, 0, len(cb.groups))
- for _, g := range cb.groups {
- rv = append(rv, g)
- }
- return rv, nil
-}
-
-func (cb *couchBackend) GetGroup(name string) (*nntp.Group, error) {
- if cb.groups == nil {
- if err := cb.fetchGroups(); err != nil {
- return nil, err
- }
- }
- g, exists := cb.groups[name]
- if !exists {
- return nil, nntpserver.ErrNoSuchGroup
- }
- return g, nil
-}
-
-func (cb *couchBackend) mkArticle(ar article) *nntp.Article {
- url := fmt.Sprintf("%s/%s/article", cb.db.DBURL(), cleanupID(ar.MsgID, true))
- return &nntp.Article{
- Header: textproto.MIMEHeader(ar.Headers),
- Body: &lazyOpener{url, nil, nil},
- Bytes: ar.Bytes,
- Lines: ar.Lines,
- }
-}
-
-func (cb *couchBackend) GetArticle(group *nntp.Group, id string) (*nntp.Article, error) {
- var ar article
- if intid, err := strconv.ParseInt(id, 10, 64); err == nil {
- results := articleResults{}
- cb.db.Query("_design/articles/_view/list", map[string]interface{}{
- "include_docs": true,
- "reduce": false,
- "key": []interface{}{group.Name, intid},
- }, &results)
-
- if len(results.Rows) != 1 {
- return nil, nntpserver.ErrInvalidArticleNumber
- }
-
- ar = results.Rows[0].Article
- } else {
- err := cb.db.Retrieve(cleanupID(id, false), &ar)
- if err != nil {
- return nil, nntpserver.ErrInvalidMessageID
- }
- }
-
- return cb.mkArticle(ar), nil
-}
-
-func (cb *couchBackend) GetArticles(group *nntp.Group,
- from, to int64) ([]nntpserver.NumberedArticle, error) {
-
- rv := make([]nntpserver.NumberedArticle, 0, 100)
-
- results := articleResults{}
- cb.db.Query("_design/articles/_view/list", map[string]interface{}{
- "include_docs": true,
- "reduce": false,
- "start_key": []interface{}{group.Name, from},
- "end_key": []interface{}{group.Name, to},
- }, &results)
-
- for _, r := range results.Rows {
- rv = append(rv, nntpserver.NumberedArticle{
- Num: int64(r.Key[1].(float64)),
- Article: cb.mkArticle(r.Article),
- })
- }
-
- return rv, nil
-}
-
-func (cb *couchBackend) AllowPost() bool {
- return true
-}
-
-func cleanupID(msgid string, escapedAt bool) string {
- s := strings.TrimFunc(msgid, func(r rune) bool {
- return r == ' ' || r == '<' || r == '>'
- })
- qe := url.QueryEscape(s)
- if escapedAt {
- return qe
- }
- return strings.Replace(qe, "%40", "@", -1)
-}
-
-func (cb *couchBackend) Post(art *nntp.Article) error {
- a := article{
- DocType: "article",
- Headers: map[string][]string(art.Header),
- Nums: make(map[string]int64),
- MsgID: cleanupID(art.Header.Get("Message-Id"), false),
- Attachments: make(map[string]*attachment),
- Added: time.Now(),
- }
-
- b := []byte{}
- buf := bytes.NewBuffer(b)
- n, err := io.Copy(buf, art.Body)
- if err != nil {
- return err
- }
- log.Printf("Read %d bytes of body", n)
-
- b = buf.Bytes()
- a.Bytes = len(b)
- a.Lines = bytes.Count(b, []byte{'\n'})
-
- a.Attachments["article"] = &attachment{"text/plain", b}
-
- for _, g := range strings.Split(art.Header.Get("Newsgroups"), ",") {
- g = strings.TrimSpace(g)
- group, err := cb.GetGroup(g)
- if err == nil {
- a.Nums[g] = atomic.AddInt64(&group.High, 1)
- atomic.AddInt64(&group.Count, 1)
- } else {
- log.Printf("Error getting group %q: %v", g, err)
- }
- }
-
- if len(a.Nums) == 0 {
- log.Printf("Found no matching groups in %v",
- art.Header["Newsgroups"])
- return nntpserver.ErrPostingFailed
- }
-
- if *optimisticPost {
- go func() {
- _, _, err = cb.db.Insert(&a)
- if err != nil {
- log.Printf("error optimistically posting article: %v", err)
- }
- }()
- } else {
- _, _, err = cb.db.Insert(&a)
- if err != nil {
- log.Printf("error posting article: %v", err)
- return nntpserver.ErrPostingFailed
- }
- }
-
- return nil
-}
-
-func (cb *couchBackend) Authorized() bool {
- return true
-}
-
-func (cb *couchBackend) Authenticate(user, pass string) (nntpserver.Backend, error) {
- return nil, nntpserver.ErrAuthRejected
-}
-
-func maybefatal(err error, f string, a ...interface{}) {
- if err != nil {
- log.Fatalf(f, a...)
- }
-}
-
-func main() {
- couchURL := flag.String("couch", "http://localhost:5984/news",
- "Couch DB.")
-
- flag.Parse()
-
- if *useSyslog {
- sl, err := syslog.New(syslog.LOG_INFO, "nntpd")
- if err != nil {
- log.Fatalf("Error initializing syslog: %v", err)
- }
- log.SetOutput(sl)
- log.SetFlags(0)
- }
-
- a, err := net.ResolveTCPAddr("tcp", ":1119")
- maybefatal(err, "Error resolving listener: %v", err)
- l, err := net.ListenTCP("tcp", a)
- maybefatal(err, "Error setting up listener: %v", err)
- defer l.Close()
-
- db, err := couch.Connect(*couchURL)
- maybefatal(err, "Can't connect to the couch: %v", err)
- err = ensureViews(&db)
- maybefatal(err, "Error setting up views: %v", err)
-
- backend := couchBackend{
- db: &db,
- }
-
- s := nntpserver.NewServer(&backend)
-
- for {
- c, err := l.AcceptTCP()
- maybefatal(err, "Error accepting connection: %v", err)
- go s.Process(c)
- }
-}
blob - 86da305cfcfa3d3637e329d51112998dd7a19366 (mode 644)
blob + /dev/null
--- examples/couchserver/encoder_test.go
+++ /dev/null
-package main
-
-import (
- "encoding/json"
- "testing"
-)
-
-func TestJSONMarshalling(t *testing.T) {
- a := attachment{
- "application/octet-stream",
- []byte("some bytes"),
- }
- b, err := json.Marshal(&a)
- if err != nil {
- t.Fatalf("Error marshalling attachment: %v", err)
- }
- t.Logf("Marshalled to %v", string(b))
-}
blob - d839bb686c590a66d3c8f79166920285e99e01a5 (mode 644)
blob + /dev/null
--- examples/couchserver/groupcreator/creator.go
+++ /dev/null
-package main
-
-import (
- "bufio"
- "flag"
- "io"
- "log"
- "os"
- "strings"
- "sync"
-
- "github.com/dustin/go-couch"
-)
-
-var wg sync.WaitGroup
-
-type agroup struct {
- Type string `json:"type"`
- Name string `json:"_id"`
- Description string `json:"description"`
-}
-
-func process(db *couch.Database, line string) {
- defer wg.Done()
- parts := strings.SplitN(strings.TrimSpace(line), " ", 2)
- if len(parts) != 2 {
- log.Printf("Error parsing %v", line)
- return
- }
- log.Printf("Processing %#v", parts)
- g := agroup{
- Type: "group",
- Name: parts[0],
- Description: parts[1],
- }
- _, _, err := db.Insert(g)
- if err != nil {
- log.Printf("Error saving %#v: %v", g, err)
- }
-}
-
-func main() {
-
- couchURL := flag.String("couch", "http://localhost:5984/news",
- "Couch DB.")
- flag.Parse()
-
- db, err := couch.Connect(*couchURL)
- if err != nil {
- log.Fatalf("Can't connect to couch: %v", err)
- }
-
- br := bufio.NewReader(os.Stdin)
- for {
- line, err := br.ReadString('\n')
- if err == io.EOF {
- break
- }
- if err != nil {
- log.Fatalf("Error reading line: %v", err)
- }
-
- wg.Add(1)
- go process(&db, line)
- }
-
- wg.Wait()
-}
blob - 7175ee798dbd54b70bc817fef65cd8b4c13901db (mode 644)
blob + /dev/null
--- examples/couchserver/lazyopen.go
+++ /dev/null
-package main
-
-import (
- "errors"
- "io"
- "io/ioutil"
- "net/http"
-)
-
-type lazyOpener struct {
- url string
- data []byte
- err error
-}
-
-func (l *lazyOpener) init() {
- res, err := http.Get(l.url)
- l.err = err
- if err == nil {
- defer res.Body.Close()
- if res.StatusCode == 200 {
- l.data, l.err = ioutil.ReadAll(res.Body)
- } else {
- l.err = errors.New(res.Status)
- }
- }
-}
-
-func (l *lazyOpener) Read(p []byte) (n int, err error) {
- if l.data == nil && l.err == nil {
- l.init()
- }
- if l.err != nil {
- return 0, err
- }
- if len(l.data) == 0 {
- return 0, io.EOF
- }
- copied := copy(p, l.data)
- l.data = l.data[copied:]
- return copied, nil
-}
blob - 6f007215f3a724c804786198deafb0e49b293ed8 (mode 644)
blob + /dev/null
--- examples/couchserver/views.go
+++ /dev/null
-package main
-
-import (
- "errors"
- "fmt"
- "log"
- "net/http"
- "strings"
-
- "github.com/dustin/go-couch"
-)
-
-const groupsjson = `{
- "_id": "_design/groups",
- "language": "javascript",
- "views": {
- "discovered": {
- "map": "function(doc) {\n if (doc.type === \"group\") {\n emit(doc._id, [doc.description, 0, 0, 0]);\n } else if (doc.type === \"article\") {\n var groups = doc.headers[\"Newsgroups\"][0].split(\",\")\n for (var i = 0; i < groups.length; i++) {\n var g = groups[i].replace(/\\s+/g, '');\n emit(g, [\"\", 1, doc.nums[g], doc.nums[g]]);\n }\n }\n}\n",
- "reduce": "function (key, values) {\n var result = [\"\", 0, 0, 0];\n\n values.forEach(function(p) {\n if (p[0].length > result[0].length) {\n result[0] = p[0];\n }\n result[1] += p[1];\n\tresult[2] = Math.min(result[2], p[2]);\n\tresult[3] = Math.max(result[3], p[3]);\n // Dumb special case\n if (result[2] === 0 && result[1] != 0) {\n result[2] = 1;\n }\n });\n\n return result;\n}"
- },
- "active": {
- "map": "function(doc) {\n if (doc.type === \"group\") {\n emit(doc._id, [doc.description, 0, 0, 0]);\n } else if (doc.type === \"article\") {\n for (var g in doc.nums) {\n emit(g, [\"\", 1, doc.nums[g], doc.nums[g]]);\n }\n }\n}\n",
- "reduce": "function (key, values) {\n var result = [\"\", 0, 0, 0];\n\n values.forEach(function(p) {\n if (p[0].length > result[0].length) {\n result[0] = p[0];\n }\n result[1] += p[1];\n\tresult[2] = Math.min(result[2], p[2]);\n\tresult[3] = Math.max(result[3], p[3]);\n // Dumb special case\n if (result[2] === 0 && result[1] != 0) {\n result[2] = 1;\n }\n });\n\n return result;\n}"
- }
- }
-}`
-
-const articlesjson = `{
- "_id": "_design/articles",
- "language": "javascript",
- "views": {
- "list": {
- "map": "function(doc) {\n if (doc.type === \"article\") {\n for (var g in doc.nums) {\n emit([g, doc.nums[g]], null);\n }\n }\n}\n",
- "reduce": "_count"
- }
- }
-}`
-
-func viewUpdateOK(i int) bool {
- return i == 200 || i == 409
-}
-
-func updateView(db *couch.Database, viewdata string) error {
- r, err := http.Post(db.DBURL(), "application/json", strings.NewReader(viewdata))
- if r == nil {
- defer r.Body.Close()
- } else {
- return err
- }
- if !viewUpdateOK(r.StatusCode) {
- return fmt.Errorf("error updating view: %v", r.Status)
- }
- return nil
-}
-
-func ensureViews(db *couch.Database) error {
- errg := updateView(db, groupsjson)
- if errg != nil {
- log.Printf("Error creating groups view %v", errg)
- }
-
- erra := updateView(db, articlesjson)
- if erra != nil {
- log.Printf("Error creating articles view %v", erra)
- }
-
- if erra != nil || errg != nil {
- return errors.New("error making views")
- }
-
- return nil
-}
blob - d362d8e50a008d020dc4a07c259cd0c16c4781c0
blob + 958970b38ad3ef4603fdd61e0e977eb5a14bd3f8
--- go.mod
+++ go.mod
module github.com/dustin/go-nntp
go 1.16
-
-require (
- github.com/dustin/go-couch v0.0.0-20160816170231-8251128dab73
- github.com/dustin/httputil v0.0.0-20170305193905-c47743f54f89 // indirect
-)
blob - 7add7f12513a6f787e71ae7387e487b7b3af1428
blob + e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
--- go.sum
+++ go.sum
-github.com/dustin/go-couch v0.0.0-20160816170231-8251128dab73 h1:YKyWSyEhJ3DYKgSpjOXpQgpxD3N+1EfIanJZj1ZEhpM=
-github.com/dustin/go-couch v0.0.0-20160816170231-8251128dab73/go.mod h1:WG/TWzFd/MRvOZ4jjna3FQ+K8AKhb2jOw4S2JMw9VKI=
-github.com/dustin/httputil v0.0.0-20170305193905-c47743f54f89 h1:A740DRjmFFdm3+GeYVfs4QN/QMOAbMw8KdsZMDhUCjQ=
-github.com/dustin/httputil v0.0.0-20170305193905-c47743f54f89/go.mod h1:ZoDWdnxro8Kesk3zrCNOHNFWtajFPSnDMjVEjGjQu/0=
