commit 00b2414c42245d4e97f287b3e6b1e889cb3c6dee from: Oliver Lowe date: Sat Jul 08 04:36:01 2023 UTC Only cache communities in filesystem This keeps things simpler commit - 4a5e46556d98c2f01ca29f5f7d42dcc46f549154 commit + 00b2414c42245d4e97f287b3e6b1e889cb3c6dee blob - e6f7d5ef0401971f6b65467886800c93d3f78c2e blob + f5681458c0573b29ade3aa333a2202deb850ffb0 --- client.go +++ client.go @@ -11,11 +11,14 @@ import ( "os" "path" "strconv" + "time" ) type Client struct { *http.Client - Address string + Address string + // If true, HTTP request summaries are printed to standard error. + Debug bool authToken string instance *url.URL ready bool @@ -56,7 +59,17 @@ func (c *Client) Communities(mode ListMode) ([]Communi } } - params := map[string]string{"type_": string(mode)} + params := map[string]string{ + "type_": string(mode), + "limit": "30", // TODO go through pages + "sort": "New", // Required - Lemmy bug - even though we don't care about sorting. + } + if mode == ListSubscribed { + if c.authToken == "" { + return nil, errors.New("not logged in, no subscriptions") + } + params["auth"] = c.authToken + } resp, err := c.get("community/list", params) if err != nil { return nil, err @@ -123,6 +136,7 @@ func (c *Client) Posts(community string, mode ListMode "community_name": community, "limit": "30", "type_": string(mode), + "sort": "New", // Required - Lemmy bug - even though we don't care about sorting. } resp, err := c.get("post/list", params) if err != nil { @@ -189,6 +203,7 @@ func (c *Client) Comments(post int, mode ListMode) ([] params := map[string]string{ "post_id": strconv.Itoa(post), "type_": string(mode), + "sort": "New", // Required - Lemmy bug - even though we don't care about sorting. } resp, err := c.get("comment/list", params) if err != nil { @@ -244,6 +259,10 @@ func (c *Client) LookupComment(id int) (Comment, error } func (c *Client) Reply(post int, parent int, msg string) error { + if c.authToken == "" { + return errors.New("not logged in") + } + params := map[string]interface{}{ "post_id": post, "content": msg, @@ -293,8 +312,15 @@ func (c *Client) get(pathname string, params map[strin return nil, err } req.Header.Set("Accept", "application/json") - fmt.Fprintf(os.Stderr, "%s %s\n", req.Method, req.URL) - return c.Do(req) + if c.Debug { + fmt.Fprintf(os.Stderr, "%s %s\n", req.Method, req.URL) + } + resp, err := c.Do(req) + if resp.StatusCode == http.StatusServiceUnavailable { + time.Sleep(2 * time.Second) + resp, err = c.get(pathname, params) + } + return resp, err } type jError struct { @@ -305,10 +331,7 @@ func (err jError) Error() string { return err.Err } func decodeError(r io.Reader) error { var jerr jError - buf := &bytes.Buffer{} - io.Copy(buf, r) - fmt.Fprintln(os.Stderr, "error", buf.String()) - if err := json.NewDecoder(buf).Decode(&jerr); err != nil { + if err := json.NewDecoder(r).Decode(&jerr); err != nil { return fmt.Errorf("decode error message: %v", err) } return jerr blob - 81d98fe53db384beb08bbc7dcb97c7e87c69a523 blob + 69f23ac4f8278328ed70ee4d0074f744c9ba5a61 --- fs.go +++ fs.go @@ -13,10 +13,9 @@ import ( ) type FS struct { - Client *Client + Client *Client + // Communities holds a cache of communities to Communities map[string]Community - Posts map[int]Post - Comments map[int]Comment root *node baseURL string started bool @@ -153,7 +152,7 @@ func (f *file) ReadDir(n int) ([]fs.DirEntry, error) { var err error if n >= len(entries) { err = io.EOF - } else if n < len(entries) { + } else { entries = entries[:n-1] } d.entryp += n @@ -178,7 +177,7 @@ func (f *dummy) Mode() fs.FileMode { func (f *dummy) ModTime() time.Time { return time.Unix(0, 0) } func (f *dummy) IsDir() bool { return f.isDir } -func (f *dummy) Sys() interface{} { return nil } +func (f *dummy) Sys() interface{} { return nil } func (fsys *FS) init() error { if fsys.Client == nil { @@ -186,13 +185,16 @@ func (fsys *FS) init() error { } if fsys.Communities == nil { fsys.Communities = make(map[string]Community) + if fsys.Client.authToken != "" { + subscribed, err := fsys.Client.Communities(ListSubscribed) + if err != nil { + return fmt.Errorf("load subscriptions: %w", err) + } + for _, c := range subscribed { + fsys.Communities[c.Name()] = c + } + } } - if fsys.Posts == nil { - fsys.Posts = make(map[int]Post) - } - if fsys.Comments == nil { - fsys.Comments = make(map[int]Comment) - } fsys.started = true return nil } @@ -203,6 +205,7 @@ func (fsys *FS) Open(name string) (fs.File, error) { } else if strings.Contains(name, `\`) { return nil, &fs.PathError{"open", name, fs.ErrInvalid} } + name = path.Clean(name) if !fsys.started { if err := fsys.init(); err != nil { @@ -248,23 +251,26 @@ func (fsys *FS) nodeOpen(node *node) (*file, error) { } func (fsys *FS) lookupNode(pathname string) (*node, error) { - root := &node{name: ".", children: []node{}} + root := &node{ + name: ".", + dummy: &dummy{ + name: ".", + isDir: true, + }, + children: []node{}, + } + for _, c := range fsys.Communities { + c := c + root.children = append(root.children, c.fsNode()) + } if pathname == "." { return root, nil } - for _, comm := range fsys.Communities { - root.children = append(root.children, comm.fsNode()) - } - if i, n := findNode(root, pathname); i >= 0 { - return n, nil - } - // First element is a community. comname, leftover, _ := strings.Cut(pathname, "/") var community Community - var ok bool - community, ok = fsys.Communities[comname] + community, ok := fsys.Communities[comname] if !ok { var err error community, err = fsys.Client.LookupCommunity(comname) @@ -273,7 +279,7 @@ func (fsys *FS) lookupNode(pathname string) (*node, er } else if err != nil { return nil, err } - fsys.Communities[community.String()] = community + fsys.Communities[community.Name()] = community root.children = append(root.children, community.fsNode()) } // punt; maybe we only want the community. @@ -287,16 +293,11 @@ func (fsys *FS) lookupNode(pathname string) (*node, er if err != nil { return nil, fmt.Errorf("get posts: %w", err) } - var post Post - post, ok = fsys.Posts[id] - if !ok { - post, err = fsys.Client.LookupPost(id) - if errors.Is(err, ErrNotFound) { - return nil, fs.ErrNotExist - } else if err != nil { - return nil, err - } - fsys.Posts[id] = post + post, err := fsys.Client.LookupPost(id) + if errors.Is(err, ErrNotFound) { + return nil, fs.ErrNotExist + } else if err != nil { + return nil, err } i, cnode := findNode(root, community.String()) if i < 0 { @@ -318,29 +319,9 @@ func (fsys *FS) lookupNode(pathname string) (*node, er if err != nil { return nil, fmt.Errorf("%s: %w", fs.ErrInvalid, err) } - var comment Comment - comment, ok = fsys.Comments[id] - if !ok { - // Reading comments can result in too many requests - // so try loading comments in bulk first. - comments, err := fsys.Client.Comments(post.ID, ListAll) - if err != nil { - return nil, err - } - for _, c := range comments { - fsys.Comments[c.ID] = c - } - // Feeling lucky? We may have a match from the recent fetch. - // Otherwise, fall back to a slow lookup. - if c, ok := fsys.Comments[id]; ok { - comment = c - } else { - comment, err = fsys.Client.LookupComment(id) - if err != nil { - return nil, err - } - fsys.Comments[id] = comment - } + comment, err := fsys.Client.LookupComment(id) + if err != nil { + return nil, err } j, pnode := findNode(root, path.Join(community.String(), postname)) @@ -408,6 +389,13 @@ func (p *Post) fsNode() node { }, }, node{ + name: "url", + dummy: &dummy{ + name: "url", + contents: []byte(p.URL), + }, + }, + node{ name: "comments", dummy: &dummy{ name: "comments", blob - 7d45b292a6c8fa92978a1ee5156234729321afbd blob + 1cb2340215e23ee9502bd8a1c4fb1c869f507c06 --- lemmy.go +++ lemmy.go @@ -23,7 +23,7 @@ func (c *Community) Size() int64 { return 0 } func (c *Community) Mode() fs.FileMode { return fs.ModeDir | 0o0555 } func (c *Community) ModTime() time.Time { return time.Unix(0, 0) } func (c *Community) IsDir() bool { return true } -func (c *Community) Sys() interface{} { return nil } +func (c *Community) Sys() interface{} { return nil } func (c Community) String() string { if c.Local { @@ -39,6 +39,7 @@ type Post struct { Title string `json:"name"` Body string CreatorID int `json:"creator_id"` + URL string // Published time.Time // Updated time.Time } @@ -52,7 +53,7 @@ func (p *Post) Size() int64 { func (p *Post) Mode() fs.FileMode { return fs.ModeDir | 0o0444 } func (p *Post) ModTime() time.Time { return time.Unix(0, 0) } func (p *Post) IsDir() bool { return true } -func (p *Post) Sys() interface{} { return nil } +func (p *Post) Sys() interface{} { return nil } type Comment struct { ID int @@ -71,7 +72,7 @@ func (c *Comment) Size() int64 { return 0 } func (c *Comment) Mode() fs.FileMode { return fs.ModeDir | 0o0444 } func (c *Comment) ModTime() time.Time { return time.Unix(0, 0) } // TODO c.Updated func (c *Comment) IsDir() bool { return true } -func (c *Comment) Sys() interface{} { return nil } +func (c *Comment) Sys() interface{} { return nil } // parseCommentPath returns the comment IDs from the path field of a Comment. func parseCommentPath(s string) ([]int, error) {