From 8cb58361a8bf582f1079d588ac6a770e2bfc3d16 Mon Sep 17 00:00:00 2001 From: Jarrett Lusso Date: Fri, 2 Oct 2026 18:06:00 -0400 Subject: [PATCH 1/5] Add a live end-to-end suite that runs the built binary against the real API with a test key. The stub tests encode what we believe the API returns, so they can't catch the API changing underneath the CLI. --- .github/workflows/live.yml | 32 +++ Makefile | 7 +- README.md | 2 + e2e/live_test.go | 488 +++++++++++++++++++++++++++++++++++++ 4 files changed, 528 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/live.yml create mode 100644 e2e/live_test.go diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml new file mode 100644 index 0000000..16a58b6 --- /dev/null +++ b/.github/workflows/live.yml @@ -0,0 +1,32 @@ +name: Live + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + live: + name: Live API (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + # Secrets aren't passed to pull requests from forks, so the tests skip + # themselves when the key is empty. + EMAILABLE_LIVE_API_KEY: ${{ secrets.EMAILABLE_LIVE_API_KEY }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Test + run: make test-live diff --git a/Makefile b/Makefile index 724c683..3f41093 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test fmt lint clean run release release-snapshot man +.PHONY: build test test-live fmt lint clean run release release-snapshot man BINARY := emailable PKG := . @@ -10,6 +10,11 @@ build: test: go test -race -coverprofile=coverage.txt ./... +# Runs the built binary against the real API. Requires EMAILABLE_LIVE_API_KEY +# set to a test key (test_...), which spends no credits. +test-live: + go test -tags live -count=1 -v ./e2e + fmt: gofmt -w . diff --git a/README.md b/README.md index 37d7702..bba0150 100644 --- a/README.md +++ b/README.md @@ -572,6 +572,8 @@ Common targets: - `make build` — compile to `bin/emailable` - `make test` — run tests with race detector and coverage +- `make test-live` — run the built binary against the real API; requires + `EMAILABLE_LIVE_API_KEY` set to a test key (`test_...`) - `make fmt` — format with `gofmt` - `make lint` — run `golangci-lint` - `make release VERSION=x.y.z` — bump `plugin.json`, commit, and tag `vx.y.z` diff --git a/e2e/live_test.go b/e2e/live_test.go new file mode 100644 index 0000000..c9e0612 --- /dev/null +++ b/e2e/live_test.go @@ -0,0 +1,488 @@ +//go:build live + +// Package e2e runs the built emailable binary against the real Emailable API +// using a test key. Test keys return simulated results and never spend +// credits, and the @example.com and .@example.com +// addresses make those results predictable. +// +// Run with: EMAILABLE_LIVE_API_KEY=test_xxx make test-live +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +const keyEnv = "EMAILABLE_LIVE_API_KEY" + +// commandTimeout bounds every invocation so a command that never finishes +// fails the test instead of hanging CI. +const commandTimeout = 90 * time.Second + +var ( + binary string + apiKey string + workDir string +) + +func TestMain(m *testing.M) { + os.Exit(run(m)) +} + +func run(m *testing.M) int { + apiKey = os.Getenv(keyEnv) + if apiKey == "" { + fmt.Fprintf(os.Stderr, "skipping live tests: %s is not set\n", keyEnv) + return 0 + } + // A live key would spend real credits on every run. + if !strings.HasPrefix(apiKey, "test_") { + fmt.Fprintf(os.Stderr, "%s must be a test key (test_...)\n", keyEnv) + return 1 + } + + tmp, err := os.MkdirTemp("", "emailable-live-") + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + defer os.RemoveAll(tmp) + + binary = filepath.Join(tmp, "emailable") + if runtime.GOOS == "windows" { + binary += ".exe" + } + build := exec.Command("go", "build", "-o", binary, ".") + build.Dir = ".." + build.Stdout = os.Stderr + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + fmt.Fprintf(os.Stderr, "build: %v\n", err) + return 1 + } + + // An empty working directory keeps a project-local .emailable/config.json + // from leaking into the run. + workDir = filepath.Join(tmp, "work") + if err := os.Mkdir(workDir, 0o755); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + if err := os.Setenv("XDG_CONFIG_HOME", filepath.Join(tmp, "config")); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + + before, err := availableCredits() + if err != nil { + fmt.Fprintf(os.Stderr, "account status: %v\n", err) + return 1 + } + + code := m.Run() + + // Test keys must never spend credits. + after, err := availableCredits() + if err != nil { + fmt.Fprintf(os.Stderr, "account status: %v\n", err) + return 1 + } + if after != before { + fmt.Fprintf(os.Stderr, "available_credits changed from %d to %d during the run\n", before, after) + return 1 + } + return code +} + +type result struct { + Stdout string + Stderr string + ExitCode int +} + +// emailable runs the binary with the test key and returns its output and exit +// code. It fails the test only if the binary could not be run or timed out. +func emailable(t *testing.T, args ...string) result { + t.Helper() + return emailableWithKey(t, apiKey, args...) +} + +func emailableWithKey(t *testing.T, key string, args ...string) result { + t.Helper() + res, err := execBinary(key, args...) + if err != nil { + t.Fatalf("emailable %s: %v\nstderr: %s", strings.Join(args, " "), err, res.Stderr) + } + return res +} + +func execBinary(key string, args ...string) (result, error) { + ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Dir = workDir + cmd.Env = append(os.Environ(), + "EMAILABLE_API_KEY="+key, + "EMAILABLE_NO_UPDATE_NOTIFIER=1", + "EMAILABLE_OUTPUT=", + "EMAILABLE_DEBUG=", + "NO_COLOR=1", + ) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + res := result{Stdout: stdout.String(), Stderr: stderr.String()} + if ctx.Err() != nil { + return res, fmt.Errorf("timed out after %s", commandTimeout) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + res.ExitCode = exitErr.ExitCode() + return res, nil + } + return res, err +} + +func availableCredits() (int, error) { + res, err := execBinary(apiKey, "account", "status", "--json") + if err != nil { + return 0, err + } + if res.ExitCode != 0 { + return 0, fmt.Errorf("exit %d: %s", res.ExitCode, res.Stderr) + } + var account struct { + AvailableCredits *int `json:"available_credits"` + } + if err := json.Unmarshal([]byte(res.Stdout), &account); err != nil { + return 0, fmt.Errorf("decode %q: %w", res.Stdout, err) + } + if account.AvailableCredits == nil { + return 0, fmt.Errorf("available_credits missing from %q", res.Stdout) + } + return *account.AvailableCredits, nil +} + +func decodeJSON(t *testing.T, s string, v any) { + t.Helper() + if err := json.Unmarshal([]byte(s), v); err != nil { + t.Fatalf("decode JSON: %v\noutput: %s", err, s) + } +} + +func requireExit(t *testing.T, res result, want int) { + t.Helper() + if res.ExitCode != want { + t.Fatalf("exit code = %d, want %d\nstdout: %s\nstderr: %s", res.ExitCode, want, res.Stdout, res.Stderr) + } +} + +type verifyResult struct { + Email string `json:"email"` + State string `json:"state"` + Reason string `json:"reason"` + Domain string `json:"domain"` + Role bool `json:"role"` + Free bool `json:"free"` + Disposable bool `json:"disposable"` + AcceptAll bool `json:"accept_all"` +} + +type batchStatus struct { + ID string `json:"id"` + Message string `json:"message"` + Emails []verifyResult `json:"emails"` + DownloadFile string `json:"download_file"` + TotalCounts map[string]int `json:"total_counts"` +} + +type expectation struct { + email string + state string + flag string // the one boolean field the address pins to true, if any +} + +// expectations covers each state plus each flag the test addresses can pin. +// Plain @ addresses leave role, free, and accept_all random, so only +// .@ addresses assert a flag. +var expectations = []expectation{ + {email: "deliverable@example.com", state: "deliverable"}, + {email: "undeliverable@example.com", state: "undeliverable"}, + {email: "risky@example.com", state: "risky"}, + {email: "unknown@example.com", state: "unknown"}, + {email: "role.undeliverable@example.com", state: "undeliverable", flag: "role"}, + {email: "free.deliverable@example.com", state: "deliverable", flag: "free"}, + {email: "disposable.risky@example.com", state: "risky", flag: "disposable"}, + {email: "accept-all.deliverable@example.com", state: "deliverable", flag: "accept_all"}, +} + +func (e expectation) check(t *testing.T, got verifyResult) { + t.Helper() + if got.Email != e.email { + t.Errorf("email = %q, want %q", got.Email, e.email) + } + if got.State != e.state { + t.Errorf("%s: state = %q, want %q", e.email, got.State, e.state) + } + if got.Reason == "" { + t.Errorf("%s: reason is empty", e.email) + } + flags := map[string]bool{ + "role": got.Role, + "free": got.Free, + "disposable": got.Disposable, + "accept_all": got.AcceptAll, + } + if e.flag != "" && !flags[e.flag] { + t.Errorf("%s: %s = false, want true", e.email, e.flag) + } +} + +func TestAccountStatus(t *testing.T) { + res := emailable(t, "account", "status", "--json") + requireExit(t, res, 0) + + var account struct { + OwnerEmail string `json:"owner_email"` + AvailableCredits *int `json:"available_credits"` + } + decodeJSON(t, res.Stdout, &account) + if !strings.Contains(account.OwnerEmail, "@") { + t.Errorf("owner_email = %q, want an email address", account.OwnerEmail) + } + if account.AvailableCredits == nil { + t.Error("available_credits is missing") + } +} + +func TestAccountStatus_InvalidKey(t *testing.T) { + res := emailableWithKey(t, "test_invalid", "account", "status", "--json") + requireExit(t, res, 2) + + var e struct { + Code string `json:"code"` + } + decodeJSON(t, res.Stderr, &e) + if e.Code != "not_authenticated" { + t.Errorf("code = %q, want not_authenticated", e.Code) + } +} + +func TestVerify(t *testing.T) { + for _, e := range expectations { + t.Run(e.email, func(t *testing.T) { + res := emailable(t, "verify", e.email, "--json") + requireExit(t, res, 0) + + var got verifyResult + decodeJSON(t, res.Stdout, &got) + e.check(t, got) + if got.Domain != "example.com" { + t.Errorf("domain = %q, want example.com", got.Domain) + } + }) + } +} + +func TestVerify_DeliverableReason(t *testing.T) { + res := emailable(t, "verify", "deliverable@example.com", "--json") + requireExit(t, res, 0) + + var got verifyResult + decodeJSON(t, res.Stdout, &got) + if got.Reason != "accepted_email" { + t.Errorf("reason = %q, want accepted_email", got.Reason) + } +} + +func TestVerify_Human(t *testing.T) { + res := emailable(t, "verify", "deliverable@example.com") + requireExit(t, res, 0) + + if !strings.Contains(res.Stdout, "deliverable@example.com") { + t.Errorf("output missing email:\n%s", res.Stdout) + } + if !strings.Contains(res.Stdout, "Deliverable") { + t.Errorf("output missing humanized state:\n%s", res.Stdout) + } +} + +// slow@example.com always returns 249, so the CLI retries and then gives up +// with try_again. +func TestVerify_Slow(t *testing.T) { + res := emailable(t, "verify", "slow@example.com", "--json") + requireExit(t, res, 3) + + var e struct { + Code string `json:"code"` + } + decodeJSON(t, res.Stderr, &e) + if e.Code != "try_again" { + t.Errorf("code = %q, want try_again", e.Code) + } +} + +func emailList(es []expectation) []string { + out := make([]string, len(es)) + for i, e := range es { + out[i] = e.email + } + return out +} + +func checkBatchEmails(t *testing.T, got batchStatus) { + t.Helper() + if len(got.Emails) != len(expectations) { + t.Fatalf("got %d emails, want %d\n%+v", len(got.Emails), len(expectations), got) + } + byEmail := make(map[string]verifyResult, len(got.Emails)) + for _, r := range got.Emails { + byEmail[r.Email] = r + } + states := map[string]int{} + for _, e := range expectations { + r, ok := byEmail[e.email] + if !ok { + t.Errorf("missing result for %s", e.email) + continue + } + e.check(t, r) + states[e.state]++ + } + if got.TotalCounts["total"] != len(expectations) { + t.Errorf("total_counts.total = %d, want %d", got.TotalCounts["total"], len(expectations)) + } + for state, n := range states { + if got.TotalCounts[state] != n { + t.Errorf("total_counts.%s = %d, want %d", state, got.TotalCounts[state], n) + } + } +} + +func submitBatch(t *testing.T, args ...string) string { + t.Helper() + res := emailable(t, append([]string{"batch", "verify", "--json"}, args...)...) + requireExit(t, res, 0) + + var submit struct { + ID string `json:"id"` + } + decodeJSON(t, res.Stdout, &submit) + if submit.ID == "" { + t.Fatalf("batch verify returned no id: %s", res.Stdout) + } + return submit.ID +} + +func TestBatch_SubmitAndGet(t *testing.T) { + id := submitBatch(t, emailList(expectations)...) + + res := emailable(t, "batch", "get", id, "--json") + requireExit(t, res, 0) + + var got batchStatus + decodeJSON(t, res.Stdout, &got) + if got.ID != id { + t.Errorf("id = %q, want %q", got.ID, id) + } + checkBatchEmails(t, got) +} + +func TestBatch_FromCSV(t *testing.T) { + path := filepath.Join(t.TempDir(), "emails.csv") + body := "email\n" + strings.Join(emailList(expectations), "\n") + "\n" + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + id := submitBatch(t, path) + + res := emailable(t, "batch", "get", id, "--json") + requireExit(t, res, 0) + + var got batchStatus + decodeJSON(t, res.Stdout, &got) + checkBatchEmails(t, got) +} + +func TestBatch_VerifyWait(t *testing.T) { + res := emailable(t, append([]string{"batch", "verify", "--wait", "--json"}, emailList(expectations)...)...) + requireExit(t, res, 0) + + var got batchStatus + decodeJSON(t, res.Stdout, &got) + checkBatchEmails(t, got) +} + +func TestBatch_GetOutputCSV(t *testing.T) { + id := submitBatch(t, emailList(expectations)...) + path := filepath.Join(t.TempDir(), "results.csv") + + res := emailable(t, "batch", "get", id, "-o", path) + requireExit(t, res, 0) + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != len(expectations)+1 { + t.Fatalf("got %d CSV lines, want %d (header plus one per email)\n%s", len(lines), len(expectations)+1, data) + } + if !strings.HasPrefix(lines[0], "email,state,") { + t.Errorf("unexpected CSV header %q", lines[0]) + } +} + +// Batches over 1,000 emails return a download_file instead of inline emails. +func TestBatch_Large(t *testing.T) { + path := filepath.Join(t.TempDir(), "emails.txt") + var b strings.Builder + for i := 0; i < 1001; i++ { + fmt.Fprintf(&b, "deliverable+%d@example.com\n", i) + } + if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { + t.Fatal(err) + } + id := submitBatch(t, path) + + res := emailable(t, "batch", "get", id, "--json") + requireExit(t, res, 0) + + var got batchStatus + decodeJSON(t, res.Stdout, &got) + if !strings.HasPrefix(got.DownloadFile, "https://") { + t.Errorf("download_file = %q, want an https URL", got.DownloadFile) + } + if len(got.Emails) != 0 { + t.Errorf("got %d inline emails, want none for a large batch", len(got.Emails)) + } + if got.TotalCounts["total"] != 1001 { + t.Errorf("total_counts.total = %d, want 1001", got.TotalCounts["total"]) + } +} + +func TestBatch_GetUnknownID(t *testing.T) { + res := emailable(t, "batch", "get", "000000000000000000000000", "--json") + requireExit(t, res, 4) + + var e struct { + Code string `json:"code"` + } + decodeJSON(t, res.Stderr, &e) + if e.Code != "not_found" { + t.Errorf("code = %q, want not_found", e.Code) + } +} From 1253dea878cdb7e1dbd8c72b1d57689d382afe55 Mon Sep 17 00:00:00 2001 From: Jarrett Lusso Date: Wed, 7 Oct 2026 10:25:49 -0400 Subject: [PATCH 2/5] Rename the suite to e2e since it runs with a test key, and trim it to a few test addresses. Also match the real API: an unknown key returns 403, and the free and accept-all flags turn a deliverable result risky. --- .github/workflows/{live.yml => e2e.yml} | 10 +-- Makefile | 8 +-- README.md | 4 +- e2e/{live_test.go => e2e_test.go} | 82 +++++++++---------------- 4 files changed, 40 insertions(+), 64 deletions(-) rename .github/workflows/{live.yml => e2e.yml} (77%) rename e2e/{live_test.go => e2e_test.go} (83%) diff --git a/.github/workflows/live.yml b/.github/workflows/e2e.yml similarity index 77% rename from .github/workflows/live.yml rename to .github/workflows/e2e.yml index 16a58b6..06ebe15 100644 --- a/.github/workflows/live.yml +++ b/.github/workflows/e2e.yml @@ -1,4 +1,4 @@ -name: Live +name: E2E on: push: @@ -10,8 +10,8 @@ permissions: contents: read jobs: - live: - name: Live API (${{ matrix.os }}) + e2e: + name: E2E (${{ matrix.os }}) strategy: fail-fast: false matrix: @@ -20,7 +20,7 @@ jobs: env: # Secrets aren't passed to pull requests from forks, so the tests skip # themselves when the key is empty. - EMAILABLE_LIVE_API_KEY: ${{ secrets.EMAILABLE_LIVE_API_KEY }} + EMAILABLE_TEST_API_KEY: ${{ secrets.EMAILABLE_TEST_API_KEY }} steps: - uses: actions/checkout@v6 @@ -29,4 +29,4 @@ jobs: go-version-file: go.mod - name: Test - run: make test-live + run: make test-e2e diff --git a/Makefile b/Makefile index 3f41093..cd03e7b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test test-live fmt lint clean run release release-snapshot man +.PHONY: build test test-e2e fmt lint clean run release release-snapshot man BINARY := emailable PKG := . @@ -10,10 +10,10 @@ build: test: go test -race -coverprofile=coverage.txt ./... -# Runs the built binary against the real API. Requires EMAILABLE_LIVE_API_KEY +# Runs the built binary against the real API. Requires EMAILABLE_TEST_API_KEY # set to a test key (test_...), which spends no credits. -test-live: - go test -tags live -count=1 -v ./e2e +test-e2e: + go test -tags e2e -count=1 -v ./e2e fmt: gofmt -w . diff --git a/README.md b/README.md index bba0150..1fc940f 100644 --- a/README.md +++ b/README.md @@ -572,8 +572,8 @@ Common targets: - `make build` — compile to `bin/emailable` - `make test` — run tests with race detector and coverage -- `make test-live` — run the built binary against the real API; requires - `EMAILABLE_LIVE_API_KEY` set to a test key (`test_...`) +- `make test-e2e` — run the built binary against the real API; requires + `EMAILABLE_TEST_API_KEY` set to a test key (`test_...`) - `make fmt` — format with `gofmt` - `make lint` — run `golangci-lint` - `make release VERSION=x.y.z` — bump `plugin.json`, commit, and tag `vx.y.z` diff --git a/e2e/live_test.go b/e2e/e2e_test.go similarity index 83% rename from e2e/live_test.go rename to e2e/e2e_test.go index c9e0612..4e977ac 100644 --- a/e2e/live_test.go +++ b/e2e/e2e_test.go @@ -1,11 +1,11 @@ -//go:build live +//go:build e2e // Package e2e runs the built emailable binary against the real Emailable API // using a test key. Test keys return simulated results and never spend // credits, and the @example.com and .@example.com // addresses make those results predictable. // -// Run with: EMAILABLE_LIVE_API_KEY=test_xxx make test-live +// Run with: EMAILABLE_TEST_API_KEY=test_xxx make test-e2e package e2e import ( @@ -23,12 +23,15 @@ import ( "time" ) -const keyEnv = "EMAILABLE_LIVE_API_KEY" +const keyEnv = "EMAILABLE_TEST_API_KEY" // commandTimeout bounds every invocation so a command that never finishes // fails the test instead of hanging CI. const commandTimeout = 90 * time.Second +// largeBatchSize is the smallest batch the API answers with a download_file. +const largeBatchSize = 1001 + var ( binary string apiKey string @@ -42,7 +45,7 @@ func TestMain(m *testing.M) { func run(m *testing.M) int { apiKey = os.Getenv(keyEnv) if apiKey == "" { - fmt.Fprintf(os.Stderr, "skipping live tests: %s is not set\n", keyEnv) + fmt.Fprintf(os.Stderr, "skipping e2e tests: %s is not set\n", keyEnv) return 0 } // A live key would spend real credits on every run. @@ -51,7 +54,7 @@ func run(m *testing.M) int { return 1 } - tmp, err := os.MkdirTemp("", "emailable-live-") + tmp, err := os.MkdirTemp("", "emailable-e2e-") if err != nil { fmt.Fprintln(os.Stderr, err) return 1 @@ -91,14 +94,15 @@ func run(m *testing.M) int { code := m.Run() - // Test keys must never spend credits. + // Test keys must never spend credits. The account may be in use + // elsewhere, so only a drop the size of the large batch counts as ours. after, err := availableCredits() if err != nil { fmt.Fprintf(os.Stderr, "account status: %v\n", err) return 1 } - if after != before { - fmt.Fprintf(os.Stderr, "available_credits changed from %d to %d during the run\n", before, after) + if before-after >= largeBatchSize { + fmt.Fprintf(os.Stderr, "available_credits dropped from %d to %d during the run\n", before, after) return 1 } return code @@ -191,14 +195,10 @@ func requireExit(t *testing.T, res result, want int) { } type verifyResult struct { - Email string `json:"email"` - State string `json:"state"` - Reason string `json:"reason"` - Domain string `json:"domain"` - Role bool `json:"role"` - Free bool `json:"free"` - Disposable bool `json:"disposable"` - AcceptAll bool `json:"accept_all"` + Email string `json:"email"` + State string `json:"state"` + Domain string `json:"domain"` + Role bool `json:"role"` } type batchStatus struct { @@ -212,21 +212,16 @@ type batchStatus struct { type expectation struct { email string state string - flag string // the one boolean field the address pins to true, if any + role bool } -// expectations covers each state plus each flag the test addresses can pin. -// Plain @ addresses leave role, free, and accept_all random, so only -// .@ addresses assert a flag. +// A plain @example.com address pins the state, and a +// .@example.com address also pins that flag. Other fields are +// random, so only these are asserted. var expectations = []expectation{ {email: "deliverable@example.com", state: "deliverable"}, {email: "undeliverable@example.com", state: "undeliverable"}, - {email: "risky@example.com", state: "risky"}, - {email: "unknown@example.com", state: "unknown"}, - {email: "role.undeliverable@example.com", state: "undeliverable", flag: "role"}, - {email: "free.deliverable@example.com", state: "deliverable", flag: "free"}, - {email: "disposable.risky@example.com", state: "risky", flag: "disposable"}, - {email: "accept-all.deliverable@example.com", state: "deliverable", flag: "accept_all"}, + {email: "role.undeliverable@example.com", state: "undeliverable", role: true}, } func (e expectation) check(t *testing.T, got verifyResult) { @@ -237,17 +232,8 @@ func (e expectation) check(t *testing.T, got verifyResult) { if got.State != e.state { t.Errorf("%s: state = %q, want %q", e.email, got.State, e.state) } - if got.Reason == "" { - t.Errorf("%s: reason is empty", e.email) - } - flags := map[string]bool{ - "role": got.Role, - "free": got.Free, - "disposable": got.Disposable, - "accept_all": got.AcceptAll, - } - if e.flag != "" && !flags[e.flag] { - t.Errorf("%s: %s = false, want true", e.email, e.flag) + if e.role && !got.Role { + t.Errorf("%s: role = false, want true", e.email) } } @@ -276,8 +262,9 @@ func TestAccountStatus_InvalidKey(t *testing.T) { Code string `json:"code"` } decodeJSON(t, res.Stderr, &e) - if e.Code != "not_authenticated" { - t.Errorf("code = %q, want not_authenticated", e.Code) + // The API answers an unknown key with 403, not 401. + if e.Code != "forbidden" { + t.Errorf("code = %q, want forbidden", e.Code) } } @@ -297,17 +284,6 @@ func TestVerify(t *testing.T) { } } -func TestVerify_DeliverableReason(t *testing.T) { - res := emailable(t, "verify", "deliverable@example.com", "--json") - requireExit(t, res, 0) - - var got verifyResult - decodeJSON(t, res.Stdout, &got) - if got.Reason != "accepted_email" { - t.Errorf("reason = %q, want accepted_email", got.Reason) - } -} - func TestVerify_Human(t *testing.T) { res := emailable(t, "verify", "deliverable@example.com") requireExit(t, res, 0) @@ -450,7 +426,7 @@ func TestBatch_GetOutputCSV(t *testing.T) { func TestBatch_Large(t *testing.T) { path := filepath.Join(t.TempDir(), "emails.txt") var b strings.Builder - for i := 0; i < 1001; i++ { + for i := 0; i < largeBatchSize; i++ { fmt.Fprintf(&b, "deliverable+%d@example.com\n", i) } if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { @@ -469,8 +445,8 @@ func TestBatch_Large(t *testing.T) { if len(got.Emails) != 0 { t.Errorf("got %d inline emails, want none for a large batch", len(got.Emails)) } - if got.TotalCounts["total"] != 1001 { - t.Errorf("total_counts.total = %d, want 1001", got.TotalCounts["total"]) + if got.TotalCounts["total"] != largeBatchSize { + t.Errorf("total_counts.total = %d, want %d", got.TotalCounts["total"], largeBatchSize) } } From f61b27804e95b291b2550ee1afc5aceb6151f1aa Mon Sep 17 00:00:00 2001 From: Jarrett Lusso Date: Wed, 7 Oct 2026 12:46:20 -0400 Subject: [PATCH 3/5] Cover more batch modes, large-batch downloads, rate limiting, and Windows in the e2e suite, and drop the credit check since test keys never spend credits. --- .github/workflows/e2e.yml | 7 +- e2e/e2e_test.go | 225 +++++++++++++++++++++++++++++++------- 2 files changed, 188 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 06ebe15..3c5e698 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -14,8 +14,11 @@ jobs: name: E2E (${{ matrix.os }}) strategy: fail-fast: false + # The jobs share the test account's rate limit, so running them one at a + # time keeps TestRateLimit's bursts from colliding. + max-parallel: 1 matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} env: # Secrets aren't passed to pull requests from forks, so the tests skip @@ -29,4 +32,4 @@ jobs: go-version-file: go.mod - name: Test - run: make test-e2e + run: go test -tags e2e -count=1 -v ./e2e diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 4e977ac..0384cae 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -9,11 +9,15 @@ package e2e import ( + "archive/zip" "bytes" "context" + "encoding/csv" "encoding/json" "errors" "fmt" + "io" + "net/http" "os" "os/exec" "path/filepath" @@ -86,26 +90,7 @@ func run(m *testing.M) int { return 1 } - before, err := availableCredits() - if err != nil { - fmt.Fprintf(os.Stderr, "account status: %v\n", err) - return 1 - } - - code := m.Run() - - // Test keys must never spend credits. The account may be in use - // elsewhere, so only a drop the size of the large batch counts as ours. - after, err := availableCredits() - if err != nil { - fmt.Fprintf(os.Stderr, "account status: %v\n", err) - return 1 - } - if before-after >= largeBatchSize { - fmt.Fprintf(os.Stderr, "available_credits dropped from %d to %d during the run\n", before, after) - return 1 - } - return code + return m.Run() } type result struct { @@ -123,14 +108,14 @@ func emailable(t *testing.T, args ...string) result { func emailableWithKey(t *testing.T, key string, args ...string) result { t.Helper() - res, err := execBinary(key, args...) + res, err := execBinary(key, nil, args...) if err != nil { t.Fatalf("emailable %s: %v\nstderr: %s", strings.Join(args, " "), err, res.Stderr) } return res } -func execBinary(key string, args ...string) (result, error) { +func execBinary(key string, env []string, args ...string) (result, error) { ctx, cancel := context.WithTimeout(context.Background(), commandTimeout) defer cancel() @@ -143,6 +128,7 @@ func execBinary(key string, args ...string) (result, error) { "EMAILABLE_DEBUG=", "NO_COLOR=1", ) + cmd.Env = append(cmd.Env, env...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -160,26 +146,6 @@ func execBinary(key string, args ...string) (result, error) { return res, err } -func availableCredits() (int, error) { - res, err := execBinary(apiKey, "account", "status", "--json") - if err != nil { - return 0, err - } - if res.ExitCode != 0 { - return 0, fmt.Errorf("exit %d: %s", res.ExitCode, res.Stderr) - } - var account struct { - AvailableCredits *int `json:"available_credits"` - } - if err := json.Unmarshal([]byte(res.Stdout), &account); err != nil { - return 0, fmt.Errorf("decode %q: %w", res.Stdout, err) - } - if account.AvailableCredits == nil { - return 0, fmt.Errorf("available_credits missing from %q", res.Stdout) - } - return *account.AvailableCredits, nil -} - func decodeJSON(t *testing.T, s string, v any) { t.Helper() if err := json.Unmarshal([]byte(s), v); err != nil { @@ -448,6 +414,46 @@ func TestBatch_Large(t *testing.T) { if got.TotalCounts["total"] != largeBatchSize { t.Errorf("total_counts.total = %d, want %d", got.TotalCounts["total"], largeBatchSize) } + if got.DownloadFile != "" { + if n := downloadedRows(t, got.DownloadFile); n != largeBatchSize { + t.Errorf("download_file has %d rows, want %d", n, largeBatchSize) + } + } +} + +// downloadedRows fetches a large batch's download_file, a zipped CSV, and +// returns its row count, excluding the header. +func downloadedRows(t *testing.T, url string) int { + t.Helper() + resp, err := http.Get(url) + if err != nil { + t.Fatalf("download: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("download: HTTP %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("download: %v", err) + } + zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("unzip: %v", err) + } + if len(zr.File) != 1 { + t.Fatalf("zip has %d files, want 1", len(zr.File)) + } + f, err := zr.File[0].Open() + if err != nil { + t.Fatalf("unzip: %v", err) + } + defer f.Close() + rows, err := csv.NewReader(f).ReadAll() + if err != nil { + t.Fatalf("read CSV: %v", err) + } + return len(rows) - 1 } func TestBatch_GetUnknownID(t *testing.T) { @@ -462,3 +468,138 @@ func TestBatch_GetUnknownID(t *testing.T) { t.Errorf("code = %q, want not_found", e.Code) } } + +func TestBatch_FromJSONWithField(t *testing.T) { + path := filepath.Join(t.TempDir(), "contacts.json") + var items []map[string]string + for _, e := range expectations { + items = append(items, map[string]string{"address": e.email}) + } + body, err := json.Marshal(items) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, body, 0o644); err != nil { + t.Fatal(err) + } + id := submitBatch(t, path, "--field", "address") + + res := emailable(t, "batch", "get", id, "--json") + requireExit(t, res, 0) + + var got batchStatus + decodeJSON(t, res.Stdout, &got) + checkBatchEmails(t, got) +} + +func TestBatch_ResponseFields(t *testing.T) { + id := submitBatch(t, append([]string{"--response-fields", "email,state"}, emailList(expectations)...)...) + + res := emailable(t, "batch", "get", id, "--json") + requireExit(t, res, 0) + + var got struct { + Emails []map[string]any `json:"emails"` + } + decodeJSON(t, res.Stdout, &got) + if len(got.Emails) != len(expectations) { + t.Fatalf("got %d emails, want %d", len(got.Emails), len(expectations)) + } + for _, e := range got.Emails { + if len(e) != 2 || e["email"] == nil || e["state"] == nil { + t.Errorf("got fields %v, want only email and state", e) + } + } +} + +// The API accepts the callback URL and retries flag without echoing them, so +// this only checks that the CLI sends them in a form the API accepts. +func TestBatch_URLAndRetries(t *testing.T) { + submitBatch(t, append([]string{"--url", "https://example.com/callback", "--retries=false"}, emailList(expectations)...)...) +} + +func TestBatch_GetPartial(t *testing.T) { + id := submitBatch(t, emailList(expectations)...) + + res := emailable(t, "batch", "get", id, "--partial", "--json") + requireExit(t, res, 0) + + var got batchStatus + decodeJSON(t, res.Stdout, &got) + checkBatchEmails(t, got) +} + +func TestBatch_GetHuman(t *testing.T) { + id := submitBatch(t, emailList(expectations)...) + + res := emailable(t, "batch", "get", id) + requireExit(t, res, 0) + for _, want := range []string{"1 Deliverable", "2 Undeliverable"} { + if !strings.Contains(res.Stdout, want) { + t.Errorf("summary missing %q:\n%s", want, res.Stdout) + } + } + + res = emailable(t, "batch", "get", id, "--all") + requireExit(t, res, 0) + for _, e := range expectations { + if !strings.Contains(res.Stdout, e.email) { + t.Errorf("--all output missing %s:\n%s", e.email, res.Stdout) + } + } +} + +func TestBatch_VerifyWaitOutputJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "results.json") + res := emailable(t, append([]string{"batch", "verify", "--wait", "-o", path}, emailList(expectations)...)...) + requireExit(t, res, 0) + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var got batchStatus + decodeJSON(t, string(data), &got) + checkBatchEmails(t, got) +} + +// rateLimitBurst is more concurrent requests than the batch status limit +// allows in one second, so some of them get a 429. The default limit is 5, +// and the test account's is 25. +const rateLimitBurst = 40 + +// TestRateLimit sends a burst of batch status requests and checks that the +// CLI retries every 429 it gets until the request succeeds. +func TestRateLimit(t *testing.T) { + id := submitBatch(t, emailList(expectations)...) + + results := make([]result, rateLimitBurst) + errs := make([]error, rateLimitBurst) + done := make(chan struct{}) + for i := range rateLimitBurst { + go func() { + results[i], errs[i] = execBinary(apiKey, []string{"EMAILABLE_DEBUG=1"}, "batch", "get", id, "--json") + done <- struct{}{} + }() + } + for range rateLimitBurst { + <-done + } + + limited := 0 + for i, res := range results { + if errs[i] != nil { + t.Fatalf("request %d: %v", i, errs[i]) + } + if res.ExitCode != 0 { + t.Errorf("request %d: exit code = %d, want 0\nstderr: %s", i, res.ExitCode, res.Stderr) + } + if strings.Contains(res.Stderr, " 429 Too Many Requests") { + limited++ + } + } + if limited == 0 { + t.Skipf("no 429 in %d concurrent requests; the account's limit may be higher", rateLimitBurst) + } + t.Logf("%d of %d requests were rate limited and retried", limited, rateLimitBurst) +} From f6705bbfaccbb80ec81ebde8c178b224afa88795 Mon Sep 17 00:00:00 2001 From: Jarrett Lusso Date: Wed, 7 Oct 2026 12:58:25 -0400 Subject: [PATCH 4/5] Check that -o saves every row of a large batch, now that `batch get -o` downloads the results file. --- e2e/e2e_test.go | 59 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 0384cae..24a5772 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -390,15 +390,7 @@ func TestBatch_GetOutputCSV(t *testing.T) { // Batches over 1,000 emails return a download_file instead of inline emails. func TestBatch_Large(t *testing.T) { - path := filepath.Join(t.TempDir(), "emails.txt") - var b strings.Builder - for i := 0; i < largeBatchSize; i++ { - fmt.Fprintf(&b, "deliverable+%d@example.com\n", i) - } - if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { - t.Fatal(err) - } - id := submitBatch(t, path) + id := submitLargeBatch(t) res := emailable(t, "batch", "get", id, "--json") requireExit(t, res, 0) @@ -603,3 +595,52 @@ func TestRateLimit(t *testing.T) { } t.Logf("%d of %d requests were rate limited and retried", limited, rateLimitBurst) } + +func submitLargeBatch(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "emails.txt") + var b strings.Builder + for i := 0; i < largeBatchSize; i++ { + fmt.Fprintf(&b, "deliverable+%d@example.com\n", i) + } + if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { + t.Fatal(err) + } + return submitBatch(t, path) +} + +// A large batch's results only exist in its download_file, so -o has to +// download them. +func TestBatch_LargeOutput(t *testing.T) { + id := submitLargeBatch(t) + dir := t.TempDir() + + csvPath := filepath.Join(dir, "results.csv") + res := emailable(t, "batch", "get", id, "-o", csvPath) + requireExit(t, res, 0) + f, err := os.Open(csvPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + rows, err := csv.NewReader(f).ReadAll() + if err != nil { + t.Fatalf("read CSV: %v", err) + } + if len(rows)-1 != largeBatchSize { + t.Errorf("CSV has %d rows, want %d", len(rows)-1, largeBatchSize) + } + + jsonPath := filepath.Join(dir, "results.json") + res = emailable(t, "batch", "get", id, "-o", jsonPath) + requireExit(t, res, 0) + data, err := os.ReadFile(jsonPath) + if err != nil { + t.Fatal(err) + } + var got batchStatus + decodeJSON(t, string(data), &got) + if len(got.Emails) != largeBatchSize { + t.Errorf("JSON has %d emails, want %d", len(got.Emails), largeBatchSize) + } +} From f21df030a4ccd2035df5652e2b868ffc19f0e399 Mon Sep 17 00:00:00 2001 From: Jarrett Lusso Date: Wed, 7 Oct 2026 15:54:44 -0400 Subject: [PATCH 5/5] Serialize e2e runs on the shared test account, ignore local API URL overrides, and bound the download with a timeout. --- .github/workflows/e2e.yml | 6 ++++++ e2e/e2e_test.go | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3c5e698..908e753 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -9,6 +9,12 @@ on: permissions: contents: read +# Every run shares the test account's rate limit, so only one run goes at a +# time. A newer pending run replaces an older pending one. +concurrency: + group: e2e-test-account + cancel-in-progress: false + jobs: e2e: name: E2E (${{ matrix.os }}) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 24a5772..0bbeef9 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -123,6 +123,10 @@ func execBinary(key string, env []string, args ...string) (result, error) { cmd.Dir = workDir cmd.Env = append(os.Environ(), "EMAILABLE_API_KEY="+key, + // Always target the real API, even if the developer points the CLI + // at a local backend. + "EMAILABLE_API_URL=", + "EMAILABLE_OAUTH_URL=", "EMAILABLE_NO_UPDATE_NOTIFIER=1", "EMAILABLE_OUTPUT=", "EMAILABLE_DEBUG=", @@ -417,7 +421,8 @@ func TestBatch_Large(t *testing.T) { // returns its row count, excluding the header. func downloadedRows(t *testing.T, url string) int { t.Helper() - resp, err := http.Get(url) + client := &http.Client{Timeout: commandTimeout} + resp, err := client.Get(url) if err != nil { t.Fatalf("download: %v", err) }