use gh sponsors api to detect dependencies

cmd/wkr-gh-sponsor/main.go

@@ -2,14 +2,13 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
 	errors "github.com/alecthomas/errors"
 	"github.com/alecthomas/kong"
-	"github.com/jpillora/backoff"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 
@@ -18,18 +17,15 @@ import (
 	"github.com/thnxdev/wkr-gh-sponsor/utils/config"
 	"github.com/thnxdev/wkr-gh-sponsor/utils/log"
 	"github.com/thnxdev/wkr-gh-sponsor/workers"
-	wkrtd "github.com/thnxdev/wkr-gh-sponsor/workers/1-td"
-	wkranimate "github.com/thnxdev/wkr-gh-sponsor/workers/2-animate"
-	wkrdonate "github.com/thnxdev/wkr-gh-sponsor/workers/3-donate"
+
+	_ "github.com/thnxdev/wkr-gh-sponsor/workers/wkr-donate"
+	_ "github.com/thnxdev/wkr-gh-sponsor/workers/wkr-entities"
+	_ "github.com/thnxdev/wkr-gh-sponsor/workers/wkr-repos"
 )
 
 // Populated during build.
 var GitCommit string = "dev"
 
-type wkr_config struct {
-	Disabled bool `help:"Disable worker." default:"false"`
-}
-
 var cli struct {
 	Version kong.VersionFlag `short:"v" help:"Print version and exit."`
 	Config  kong.ConfigFlag  `short:"C" help:"Config file." placeholder:"FILE" env:"CONFIG_PATH"`
@@ -38,20 +34,15 @@ var cli struct {
 	LogJSON  bool         `help:"Log in JSON format." group:"Observability:"`
 
 	DbPath               string                     `help:"Path to db file." required:"" env:"DB_PATH" default:"db.sql"`
-	TdApiUrl             wkrghsponsor.TdApiUrl      `help:"thanks.dev API URL." required:"" env:"TD_API_URL" default:"https://api.thanks.dev/v1/deps"`
-	TdApiKey             wkrghsponsor.TdApiKey      `help:"thanks.dev API key." required:"" env:"TD_API_KEY"`
 	GhClassicAccessToken wkrghsponsor.GhAccessToken `help:"GitHub classis access token with admin:org & user scopes." required:"" env:"GH_CLASSIC_ACCESS_TOKEN"`
 
 	Entities []wkrghsponsor.Entity `help:"The GitHub entities to process sponsorships for. First entity in the list is considered DEFAULT." required:""`
 
 	SponsorAmount wkrghsponsor.SponsorAmount `help:"The amount to donate to each dependency" default:"1"`
-
-	WkrTd      wkr_config `embed:"" prefix:"wkr-td-"`
-	WkrAnimate wkr_config `embed:"" prefix:"wkr-animate-"`
-	WkrDonate  wkr_config `embed:"" prefix:"wkr-donate-"`
 }
 
 func main() {
+
 	options := []kong.Option{
 		kong.Configuration(config.CreateLoader),
 		kong.HelpOptions{Compact: true},
@@ -66,6 +57,14 @@ func main() {
 			"version": GitCommit,
 		},
 	}
+
+	wkrs := workers.GetWorkers()
+	for name, cfg := range wkrs {
+		options = append(options,
+			kong.Embed(cfg, fmt.Sprintf(`embed:"" prefix:"%s-" group:"%s"`, name, name)),
+			kong.Bind(cfg))
+	}
+
 	kctx := kong.Parse(&cli, options...)
 
 	logger := logrus.New()
@@ -79,20 +78,12 @@ func main() {
 
 	ctx := log.LoggerContext(context.Background(), logger)
 
-	cleanup := Cleanup{}
-
 	ctx, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM)
 	wg, ctx := errgroup.WithContext(ctx)
 	kctx.Exit = func(code int) {
 		kctx.Exit = os.Exit
 		stop()
 		err := wg.Wait()
-		for _, fn := range cleanup {
-			err := fn()
-			if err != nil {
-				kctx.Errorf("wkr-gh-sponsor: error: shutdown failed: %v", err)
-			}
-		}
 		if err != nil && !errors.Is(err, context.Canceled) {
 			kctx.FatalIfErrorf(err)
 		}
@@ -104,100 +95,13 @@ func main() {
 	kctx.FatalIfErrorf(err)
 
 	kctx.Bind(db)
-	kctx.Bind(cli.TdApiUrl)
-	kctx.Bind(cli.TdApiKey)
 	kctx.Bind(cli.GhClassicAccessToken)
 	kctx.Bind(cli.Entities)
 	kctx.Bind(cli.SponsorAmount)
 
-	// Start the workers
-	wkrs := []struct {
-		name         string
-		fn           any
-		disabled     bool
-		workInterval time.Duration
-	}{
-		{
-			name:         "wkr-td",
-			fn:           wkrtd.New,
-			disabled:     cli.WkrTd.Disabled,
-			workInterval: time.Hour * 3, // 4 times a day
-		},
-		{
-			name:         "wkr-animate",
-			fn:           wkranimate.New,
-			disabled:     cli.WkrAnimate.Disabled,
-			workInterval: time.Minute, // once a minute
-		},
-		{
-			name:         "wkr-donate",
-			fn:           wkrdonate.New,
-			disabled:     cli.WkrDonate.Disabled,
-			workInterval: time.Minute, // once a minute
-		},
-	}
-	for _, w := range wkrs {
-		w := w
-
-		logger := logger.WithField("name", w.name)
-		ctx := log.LoggerContext(ctx, logger)
-
-		if w.disabled {
-			logger.Infof("Worker %s not enabled", w.name)
-			continue
-		}
-
-		out, err := kctx.Call(w.fn)
-		if err != nil {
-			logger.WithError(err).Error("failed to start")
-		}
-
-		wkr := out[0].(workers.Worker)
+	workers.Run(ctx, wg, kctx)
 
-		wg.Go(func() error {
-			logger.Info("Starting")
-			retry := &backoff.Backoff{
-				Min:    w.workInterval,
-				Factor: 1.1,
-				Jitter: true,
-				Max:    w.workInterval * 4,
-			}
-			for {
-				logger.Info("Running")
-
-				// default loop interval is once an hour
-				// but if the worker returns an error we'll
-				// do a backoff retry in one minute up to 4 mins
-				delay := w.workInterval
-
-				err := wkr(ctx)
-				if err != nil {
-					delay = retry.Duration()
-				} else {
-					retry.Reset()
-				}
-
-				select {
-				case <-ctx.Done():
-					if errors.Is(err, context.Canceled) {
-						return nil
-					}
-					return ctx.Err()
-				case <-time.After(delay):
-				}
-			}
-		})
-	}
-
-	err = wg.Wait()
-	if err != nil && !errors.Is(err, context.Canceled) {
-		kctx.FatalIfErrorf(err)
-	}
+	logger.Info("exiting")
 
 	kctx.Exit(0)
 }
-
-// Cleanup is a convenience type for registering cleanup functions.
-type Cleanup []func() error
-
-func (c *Cleanup) Add(f func() error) { *c = append(*c, f) }

database/queries/wkrdonate.sql

@@ -0,0 +1,36 @@
+-- name: GetDonables :many
+
+SELECT id, sponsor_id, recipient_id
+FROM donations
+WHERE
+	donate_ts < last_ts AND
+	donate_attempt_ts < UNIXEPOCH() - 3600;
+
+-- name: GetAreDonatesFinished :one
+
+WITH
+	num_repos_remaining AS (
+		SELECT COUNT(rowid) AS num
+		FROM repos
+		WHERE animate_ts < last_ts
+	)
+SELECT
+	(
+		num_repos_remaining.num = 0 AND
+		kv.v IS NOT NULL
+	) AS is_finished
+FROM num_repos_remaining
+LEFT JOIN kvstore kv ON k = 'entity-ts';
+
+-- name: UpdateDonationDonateAttemptTs :exec
+
+UPDATE donations
+SET donate_attempt_ts = UNIXEPOCH()
+WHERE id = ?;
+
+-- name: UpdateDonationDonateTs :exec
+
+UPDATE donations
+SET donate_ts = UNIXEPOCH()
+WHERE id = ?;
+

database/queries/wkrentities.sql

@@ -0,0 +1,33 @@
+-- name: KvstoreGetLastStatus :one
+
+WITH d AS (VALUES(1))
+SELECT
+	kvc.v AS next_page,
+	kvt.v AS entity_ts
+FROM d
+LEFT JOIN kvstore kvc ON kvc.k = 'next-page'
+LEFT JOIN kvstore kvt ON kvt.k = 'entity-ts';
+
+-- name: ReposInsert :exec
+
+INSERT INTO repos (owner_name, repo_name, last_ts)
+VALUES (?, ?, UNIXEPOCH())
+ON CONFLICT (owner_name, repo_name)
+DO NOTHING;
+
+-- name: KvstoreInsertNextPage :exec
+
+INSERT INTO kvstore (k, v)
+VALUES ('next-page', ?)
+ON CONFLICT (k)
+DO UPDATE
+	SET v = EXCLUDED.v;
+
+-- name: KvstoreInsertEntityTs :exec
+
+INSERT INTO kvstore (k, v)
+VALUES ('entity-ts', ?)
+ON CONFLICT (k)
+DO UPDATE
+	SET v = EXCLUDED.v;
+