subculture-collective/discord-spywatcher
1
0
Fork 0
Code Issues 2 Pull Requests 23 Actions Packages Projects Releases Wiki Activity

[WIP] Create contributing guidelines for open source contributors (#170)

Browse Source 
* Initial plan

* docs: add comprehensive contributing guidelines and templates

Co-authored-by: onnwee <211922112+onnwee@users.noreply.github.com>

* docs: update README and SECURITY with better formatting and links

Co-authored-by: onnwee <211922112+onnwee@users.noreply.github.com>

* docs: finalize contributing guidelines and formatting

Co-authored-by: onnwee <211922112+onnwee@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: onnwee <211922112+onnwee@users.noreply.github.com>
This commit was merged in pull request #170.
This commit is contained in:
Copilot
2025-11-04 15:38:59 -06:00
committed by GitHub
parent ba81c0a8dc
commit 2aa4be44f7
33 changed files with 3288 additions and 1769 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
.github/ISSUE_TEMPLATE/bug_report.yml vendored Normal file
View File

@@ -0,0 +1,132 @@
name: Bug Report
description: Report a bug or unexpected behavior
title: "[Bug]: "
labels: ["bug", "needs-triage"]
assignees: []
body:
- type: markdown
attributes:
value: |
Thanks for taking the time to report a bug! Please fill out the information below to help us resolve the issue.
- type: checkboxes
id: preflight
attributes:
label: Preflight Checklist
description: Please ensure you've completed these steps before submitting your bug report.
options:
- label: I have searched existing issues to ensure this bug hasn't already been reported
required: true
- label: I have read the [documentation](https://github.com/subculture-collective/discord-spywatcher/blob/main/README.md)
required: true
- label: I am using the latest version of the software
required: false
- type: textarea
id: description
attributes:
label: Description
description: A clear and concise description of the bug
placeholder: What happened?
validations:
required: true
- type: textarea
id: expected
attributes:
label: Expected Behavior
description: What did you expect to happen?
placeholder: Describe the expected behavior
validations:
required: true
- type: textarea
id: actual
attributes:
label: Actual Behavior
description: What actually happened?
placeholder: Describe what actually happened
validations:
required: true
- type: textarea
id: reproduction
attributes:
label: Steps to Reproduce
description: How can we reproduce this issue?
placeholder: |
1. Go to '...'
2. Click on '...'
3. Scroll down to '...'
4. See error
validations:
required: true
- type: dropdown
id: component
attributes:
label: Component
description: Which component is affected?
options:
- Backend (API/Bot)
- Frontend (Dashboard)
- Database
- Docker/Deployment
- Documentation
- SDK
- Plugin System
- Other
validations:
required: true
- type: input
id: version
attributes:
label: Version
description: What version of the software are you running?
placeholder: v1.0.0 or commit SHA
validations:
required: false
- type: dropdown
id: environment
attributes:
label: Environment
description: What environment are you running in?
options:
- Development (local)
- Docker (development)
- Docker (production)
- Kubernetes
- Other
validations:
required: true
- type: textarea
id: logs
attributes:
label: Relevant Logs
description: Please paste any relevant logs or error messages
render: shell
placeholder: |
Paste your logs here...
validations:
required: false
- type: textarea
id: context
attributes:
label: Additional Context
description: Add any other context about the problem here
placeholder: Environment variables, configuration, screenshots, etc.
validations:
required: false
- type: checkboxes
id: contribution
attributes:
label: Contribution
description: Would you like to work on fixing this issue?
options:
- label: I would be willing to submit a PR to fix this issue
required: false

11
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@@ -0,0 +1,11 @@
blank_issues_enabled: true
contact_links:
- name: 💬 Discussions
url: https://github.com/subculture-collective/discord-spywatcher/discussions
about: Ask questions, share ideas, and engage with the community
- name: 📚 Documentation
url: https://github.com/subculture-collective/discord-spywatcher/blob/main/README.md
about: Read the full documentation and guides
- name: 🔐 Security Issues
url: https://github.com/subculture-collective/discord-spywatcher/security/advisories/new
about: Report security vulnerabilities privately

94
.github/ISSUE_TEMPLATE/documentation.yml vendored Normal file
View File

@@ -0,0 +1,94 @@
name: Documentation Improvement
description: Suggest improvements to documentation
title: "[Docs]: "
labels: ["documentation", "needs-triage"]
assignees: []
body:
- type: markdown
attributes:
value: |
Thanks for helping improve our documentation! Clear docs help everyone.
- type: checkboxes
id: preflight
attributes:
label: Preflight Checklist
description: Please ensure you've completed these steps before submitting.
options:
- label: I have searched existing issues to ensure this hasn't been reported
required: true
- type: dropdown
id: doc-type
attributes:
label: Documentation Type
description: What type of documentation needs improvement?
options:
- README
- Contributing Guidelines
- API Documentation
- User Guide
- Developer Guide
- Setup Instructions
- Code Comments
- Configuration Examples
- Troubleshooting Guide
- Other
validations:
required: true
- type: input
id: location
attributes:
label: Documentation Location
description: Where is the documentation that needs improvement?
placeholder: e.g., README.md, docs/api/README.md, backend/src/auth.ts
validations:
required: true
- type: textarea
id: issue
attributes:
label: Issue with Current Documentation
description: What's wrong, missing, or unclear in the current documentation?
placeholder: |
Examples:
- Information is outdated
- Missing explanation of X
- Instructions are unclear
- Broken links
- Missing code examples
- Typos or grammatical errors
validations:
required: true
- type: textarea
id: improvement
attributes:
label: Suggested Improvement
description: How should the documentation be improved?
placeholder: |
Provide specific suggestions for improvement:
- What information should be added?
- How should it be reorganized?
- What examples would help?
validations:
required: true
- type: textarea
id: context
attributes:
label: Additional Context
description: Add any other context, screenshots, or examples
placeholder: Links to similar documentation, screenshots of errors, etc.
validations:
required: false
- type: checkboxes
id: contribution
attributes:
label: Contribution
description: Would you like to work on improving this documentation?
options:
- label: I would be willing to submit a PR to improve this documentation
required: false

120
.github/ISSUE_TEMPLATE/feature_request.yml vendored Normal file
View File

@@ -0,0 +1,120 @@
name: Feature Request
description: Suggest a new feature or enhancement
title: "[Feature]: "
labels: ["enhancement", "needs-triage"]
assignees: []
body:
- type: markdown
attributes:
value: |
Thanks for taking the time to suggest a new feature! We appreciate your input.
- type: checkboxes
id: preflight
attributes:
label: Preflight Checklist
description: Please ensure you've completed these steps before submitting your feature request.
options:
- label: I have searched existing issues and discussions to ensure this hasn't been suggested before
required: true
- label: I have read the [documentation](https://github.com/subculture-collective/discord-spywatcher/blob/main/README.md)
required: true
- type: textarea
id: problem
attributes:
label: Problem Statement
description: Is your feature request related to a problem? Please describe.
placeholder: I'm frustrated when... or I need to...
validations:
required: true
- type: textarea
id: solution
attributes:
label: Proposed Solution
description: Describe the solution you'd like to see
placeholder: A clear and concise description of what you want to happen
validations:
required: true
- type: textarea
id: alternatives
attributes:
label: Alternatives Considered
description: Have you considered any alternative solutions or features?
placeholder: Describe any alternative solutions or features you've considered
validations:
required: false
- type: dropdown
id: component
attributes:
label: Component
description: Which component would this feature affect?
options:
- Backend (API/Bot)
- Frontend (Dashboard)
- Database
- Docker/Deployment
- Documentation
- SDK
- Plugin System
- Infrastructure
- Other
validations:
required: true
- type: dropdown
id: priority
attributes:
label: Priority
description: How important is this feature to you?
options:
- Critical - Blocking my usage
- High - Would significantly improve my workflow
- Medium - Would be nice to have
- Low - Just an idea
validations:
required: true
- type: textarea
id: use-case
attributes:
label: Use Case
description: Describe your use case and how this feature would benefit you and others
placeholder: |
As a [role], I want to [action] so that [benefit]
Example scenarios:
- Scenario 1: ...
- Scenario 2: ...
validations:
required: true
- type: textarea
id: mockups
attributes:
label: Mockups or Examples
description: If applicable, add mockups, screenshots, or examples to help explain your feature
placeholder: Drag and drop images here, or describe what you envision
validations:
required: false
- type: textarea
id: context
attributes:
label: Additional Context
description: Add any other context about the feature request here
placeholder: Links to similar features, technical considerations, etc.
validations:
required: false
- type: checkboxes
id: contribution
attributes:
label: Contribution
description: Would you like to work on implementing this feature?
options:
- label: I would be willing to submit a PR to implement this feature
required: false

98
.github/PULL_REQUEST_TEMPLATE.md vendored Normal file
View File

@@ -0,0 +1,98 @@
## Description
<!-- Provide a clear and concise description of your changes -->
## Related Issues
<!-- Link to any related issues using #issue_number -->
<!-- Example: Fixes #123, Closes #456, Related to #789 -->
## Type of Change
<!-- Please check the one that applies to this PR using "x" -->
- [ ] 🐛 Bug fix (non-breaking change which fixes an issue)
- [ ] ✨ New feature (non-breaking change which adds functionality)
- [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [ ] 📝 Documentation update
- [ ] 🎨 Code style update (formatting, renaming)
- [ ] ♻️ Refactoring (no functional changes, no API changes)
- [ ] ⚡ Performance improvement
- [ ] ✅ Test update
- [ ] 🔧 Build/configuration update
- [ ] 🔒 Security fix
## Changes Made
<!-- Provide a detailed list of changes -->
-
-
-
## Testing
<!-- Describe the testing you've done -->
### Test Environment
- [ ] Local development
- [ ] Docker (development)
- [ ] Docker (production)
- [ ] Kubernetes
### Tests Performed
<!-- Describe what you tested and the results -->
- [ ] Unit tests pass locally
- [ ] Integration tests pass locally
- [ ] Manual testing completed
- [ ] All existing tests pass
### Test Coverage
<!-- If applicable, include test coverage information -->
## Screenshots
<!-- If applicable, add screenshots to help explain your changes -->
## Checklist
<!-- Please check all that apply using "x" -->
- [ ] My code follows the project's style guidelines
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my feature works
- [ ] New and existing unit tests pass locally with my changes
- [ ] Any dependent changes have been merged and published
- [ ] I have checked my code and corrected any misspellings
- [ ] My commit messages follow the [Conventional Commits](https://www.conventionalcommits.org/) specification
## Additional Notes
<!-- Add any additional notes, context, or information that reviewers should know -->
## Breaking Changes
<!-- If this PR includes breaking changes, describe them here and provide migration instructions -->
## Performance Impact
<!-- If applicable, describe any performance implications of your changes -->
## Security Considerations
<!-- If applicable, describe any security implications or considerations -->
## Deployment Notes
<!-- If applicable, describe any special deployment considerations or steps required -->
## Reviewer Notes
<!-- Optional: Add any specific areas you'd like reviewers to focus on -->

2
.gitignore vendored
View File

@@ -99,4 +99,4 @@ nginx/ssl/
loki/data/
grafana/data/
*.zip
*.zip

48
ACCESSIBILITY.md
View File

@@ -9,27 +9,31 @@ Discord Spywatcher is committed to providing an inclusive experience for all use
## Keyboard Navigation
### Global Navigation
- **Skip to Main Content**: Press `Tab` on page load to reveal a "Skip to main content" link that allows bypassing navigation
- **Tab Order**: All interactive elements follow a logical tab order
- **Focus Indicators**: Visible focus indicators (blue outline) on all interactive elements
- **No Keyboard Traps**: Users can navigate in and out of all components using keyboard alone
### Keyboard Shortcuts
| Key | Action |
|-----|--------|
| `Tab` | Navigate to next focusable element |
| `Shift + Tab` | Navigate to previous focusable element |
| `Enter` / `Space` | Activate buttons and links |
| `Escape` | Close modals and dialogs (when implemented) |
| Key | Action |
| ----------------- | ------------------------------------------- |
| `Tab` | Navigate to next focusable element |
| `Shift + Tab` | Navigate to previous focusable element |
| `Enter` / `Space` | Activate buttons and links |
| `Escape` | Close modals and dialogs (when implemented) |
## Screen Reader Support
### Semantic HTML
- Proper heading hierarchy (h1 → h2 → h3, etc.)
- Semantic landmarks: `<header>`, `<main>`, `<nav>`, `<aside>`, `<section>`
- Native HTML elements used where possible (`<button>`, `<a>`, etc.)
### ARIA Attributes
We use ARIA attributes to enhance accessibility:
- **aria-label**: Provides accessible names for icon-only buttons and interactive elements
@@ -41,13 +45,16 @@ We use ARIA attributes to enhance accessibility:
- **aria-hidden**: Hides decorative elements from screen readers
### Tables
All data tables include:
- `<caption>` for table descriptions
- `scope="col"` and `scope="row"` for headers
- `<abbr>` for abbreviated column headers
- Alternative text descriptions for complex data
### Charts & Visualizations
- Visual charts include `role="img"` with descriptive `aria-label`
- Hidden data tables provided as fallback for screen readers
- Summary statistics announced via `aria-live` regions
@@ -55,17 +62,21 @@ All data tables include:
## Visual Accessibility
### Color Contrast
All text meets WCAG AA contrast requirements:
- Normal text: 4.5:1 contrast ratio
- Large text (18pt+): 3.0:1 contrast ratio
- UI components: 3.0:1 contrast ratio
### Text Scaling
- Support for 200% zoom without horizontal scrolling
- Responsive text sizing using relative units (rem, em)
- No fixed pixel-based font sizes
### Color Independence
- Information is not conveyed by color alone
- Icons and patterns supplement color coding
- Status indicators include text labels
@@ -73,12 +84,14 @@ All text meets WCAG AA contrast requirements:
## Forms & Inputs
### Form Accessibility
- All inputs have associated `<label>` elements
- Required fields indicated with visual and screen-reader accessible markers
- Error messages clearly associated with their fields via `aria-describedby`
- Validation feedback announced to screen readers
### Input Assistance
- Clear instructions for expected input formats
- Inline validation with helpful error messages
- Success feedback when forms are submitted
@@ -86,11 +99,13 @@ All text meets WCAG AA contrast requirements:
## Dynamic Content
### Live Regions
- Notifications use `role="status"` with `aria-live="polite"`
- Statistics use `aria-live="polite"` for automatic updates
- Critical alerts use `aria-live="assertive"` (when implemented)
### Focus Management
- Focus automatically moved to modals when opened (when implemented)
- Focus returned to triggering element when modal closed (when implemented)
- Focus moved to error fields on validation failure
@@ -98,23 +113,28 @@ All text meets WCAG AA contrast requirements:
## Testing
### Automated Testing
We use the following tools for automated accessibility testing:
- **vitest-axe**: Integrated into our test suite to catch accessibility violations
- **ESLint jsx-a11y plugin**: Catches common accessibility issues during development
Run accessibility tests:
```bash
npm run test -- src/__tests__/accessibility
```
### Manual Testing
We recommend testing with:
- **Keyboard-only navigation**: Tab through the entire application
- **Screen readers**:
- NVDA (Windows) - Free
- JAWS (Windows) - Commercial
- VoiceOver (macOS/iOS) - Built-in
- TalkBack (Android) - Built-in
- NVDA (Windows) - Free
- JAWS (Windows) - Commercial
- VoiceOver (macOS/iOS) - Built-in
- TalkBack (Android) - Built-in
- **Browser zoom**: Test at 200% zoom level
- **High contrast mode**: Windows High Contrast Mode
@@ -137,10 +157,10 @@ If you experience any accessibility issues or have suggestions for improvement:
1. **GitHub Issues**: Open an issue at [discord-spywatcher/issues](https://github.com/subculture-collective/discord-spywatcher/issues)
2. **Label**: Use the `accessibility` label
3. **Include**:
- Browser and version
- Assistive technology (if applicable)
- Steps to reproduce
- Expected vs. actual behavior
- Browser and version
- Assistive technology (if applicable)
- Steps to reproduce
- Expected vs. actual behavior
## Best Practices for Contributors

94
ADVANCED_VISUALIZATIONS.md
View File

@@ -7,68 +7,74 @@ The Discord Spywatcher application now includes advanced visualization capabilit
## Features
### 1. Network/Relationship Graph
- **Technology**: Vis-network
- **Purpose**: Visualizes relationships between users and channels as an interactive network
- **Features**:
- Interactive node dragging and zooming
- Node size represents activity level (suspicion + ghost scores)
- Color-coded nodes (users in blue, channels in green)
- Hover tooltips showing detailed information
- Physics-based layout for natural clustering
- Real-time updates
- Interactive node dragging and zooming
- Node size represents activity level (suspicion + ghost scores)
- Color-coded nodes (users in blue, channels in green)
- Hover tooltips showing detailed information
- Physics-based layout for natural clustering
- Real-time updates
### 2. Sankey Flow Diagram
- **Technology**: D3.js with d3-sankey
- **Purpose**: Shows the flow of interactions from users to channels
- **Features**:
- Flow width represents interaction volume
- Left side shows users, right side shows channels
- Color-coded by source entity type
- Hover tooltips with interaction counts
- Smooth, animated transitions
- Responsive layout
- Flow width represents interaction volume
- Left side shows users, right side shows channels
- Color-coded by source entity type
- Hover tooltips with interaction counts
- Smooth, animated transitions
- Responsive layout
### 3. Chord Diagram
- **Technology**: D3.js chord layout
- **Purpose**: Displays circular interaction patterns between all entities
- **Features**:
- Circular layout showing all relationships
- Arc width represents total interactions
- Ribbons show interaction strength between entities
- Color-coded by entity
- Interactive hover effects
- Compact visualization for complex relationships
- Circular layout showing all relationships
- Arc width represents total interactions
- Ribbons show interaction strength between entities
- Color-coded by entity
- Interactive hover effects
- Compact visualization for complex relationships
### 4. Interactive Filtering
- **Purpose**: Real-time data filtering and exploration
- **Features**:
- Suspicion score range filter (0-100)
- Ghost score range filter (0-100)
- Minimum interactions threshold
- User search by name
- Channel search by name
- Active filter count indicator
- One-click filter reset
- Filters apply to all visualizations
- Suspicion score range filter (0-100)
- Ghost score range filter (0-100)
- Minimum interactions threshold
- User search by name
- Channel search by name
- Active filter count indicator
- One-click filter reset
- Filters apply to all visualizations
### 5. Chart Export
- **Technology**: html2canvas
- **Purpose**: Export visualizations as PNG images
- **Features**:
- High-quality 2x resolution export
- Automatic filename with timestamp
- Dark theme preserved in export
- One-click download
- Works with all chart types
- High-quality 2x resolution export
- Automatic filename with timestamp
- Dark theme preserved in export
- One-click download
- Works with all chart types
### 6. Drill-Down Panel
- **Purpose**: Detailed entity information modal
- **Features**:
- User and channel detail views
- Metrics display (suspicion, ghost scores, message counts)
- Recent activity timeline
- Smooth animations
- Click outside to close
- User and channel detail views
- Metrics display (suspicion, ghost scores, message counts)
- Recent activity timeline
- Smooth animations
- Click outside to close
## Usage
@@ -81,6 +87,7 @@ The Discord Spywatcher application now includes advanced visualization capabilit
### Switching Between Visualizations
Use the view toggle buttons at the top of the page:
- **Network Graph**: Best for understanding relationships
- **Sankey Flow**: Best for tracking interaction flows
- **Chord Diagram**: Best for comparing all interactions
@@ -105,13 +112,13 @@ Use the view toggle buttons at the top of the page:
```json
{
"d3": "^7.9.0",
"d3-sankey": "^0.12.3",
"@types/d3": "^7.4.3",
"@types/d3-sankey": "^0.12.4",
"vis-network": "^10.0.2",
"vis-data": "^8.0.3",
"html2canvas": "^1.4.1"
"d3": "^7.9.0",
"d3-sankey": "^0.12.3",
"@types/d3": "^7.4.3",
"@types/d3-sankey": "^0.12.4",
"vis-network": "^10.0.2",
"vis-data": "^8.0.3",
"html2canvas": "^1.4.1"
}
```
@@ -249,6 +256,7 @@ Potential additions for future versions:
## Support
For issues or questions:
- Check existing GitHub issues
- Review the main [README.md](./README.md)
- Consult [CONTRIBUTING.md](./CONTRIBUTING.md)

105
ANALYTICS.md
View File

@@ -7,6 +7,7 @@ Discord SpyWatcher includes a comprehensive, GDPR-compliant analytics system for
## Features
### 📊 Tracking Capabilities
- **User Behavior Tracking**: Page views, button clicks, feature usage
- **Performance Metrics**: API response times, database query performance
- **Feature Analytics**: Track which features are most used
@@ -14,6 +15,7 @@ Discord SpyWatcher includes a comprehensive, GDPR-compliant analytics system for
- **Error Tracking**: Automatic error event capture
### 🔒 Privacy & Compliance
- **GDPR Compliant**: Full compliance with European data protection regulations
- **Consent Management**: User opt-in/opt-out support
- **Data Anonymization**: Automatic hashing of sensitive data when consent not given
@@ -21,6 +23,7 @@ Discord SpyWatcher includes a comprehensive, GDPR-compliant analytics system for
- **Data Retention**: Configurable retention policies
### 📈 Dashboard & Insights
- Real-time analytics dashboard
- User activity metrics
- Feature usage statistics
@@ -32,6 +35,7 @@ Discord SpyWatcher includes a comprehensive, GDPR-compliant analytics system for
### Backend Components
#### Database Models
```prisma
model UserAnalyticsEvent {
id String @id @default(cuid())
@@ -79,9 +83,11 @@ model AnalyticsSummary {
```
#### Analytics Service
Location: `backend/src/services/analytics.ts`
**Key Functions:**
- `trackEvent()` - Track any analytics event
- `trackFeatureUsage()` - Track feature usage
- `trackPerformance()` - Track performance metrics
@@ -92,6 +98,7 @@ Location: `backend/src/services/analytics.ts`
- `cleanOldAnalyticsData()` - Clean data based on retention policy
**Event Types:**
```typescript
enum AnalyticsEventType {
PAGE_VIEW = 'PAGE_VIEW',
@@ -106,9 +113,11 @@ enum AnalyticsEventType {
```
#### Middleware
Location: `backend/src/middleware/analyticsTracking.ts`
Automatically tracks:
- All API requests
- Response times
- Error events
@@ -118,30 +127,35 @@ Automatically tracks:
**GET /api/metrics/summary**
Get analytics summary for a date range
```bash
GET /api/metrics/summary?startDate=2024-01-01&endDate=2024-01-07&metric=active_users
```
**GET /api/metrics/features**
Get feature usage statistics
```bash
GET /api/metrics/features?startDate=2024-01-01&endDate=2024-01-07
```
**GET /api/metrics/activity**
Get user activity metrics
```bash
GET /api/metrics/activity?startDate=2024-01-01&endDate=2024-01-07
```
**GET /api/metrics/performance**
Get performance metrics
```bash
GET /api/metrics/performance?type=API_RESPONSE_TIME&startDate=2024-01-01
```
**POST /api/metrics/event**
Track a custom event from frontend
```bash
POST /api/metrics/event
{
@@ -153,6 +167,7 @@ POST /api/metrics/event
**GET /api/metrics/dashboard**
Get comprehensive dashboard data
```bash
GET /api/metrics/dashboard
```
@@ -160,11 +175,18 @@ GET /api/metrics/dashboard
### Frontend Components
#### Analytics Service
Location: `frontend/src/lib/analytics.ts`
**Tracking Functions:**
```typescript
import { trackEvent, trackPageView, trackButtonClick, trackFeatureUsage } from '../lib/analytics';
import {
trackEvent,
trackPageView,
trackButtonClick,
trackFeatureUsage,
} from '../lib/analytics';
// Track a page view
trackPageView('/dashboard');
@@ -177,16 +199,17 @@ trackFeatureUsage('ghost_analysis', { userCount: 50 });
```
**Consent Management:**
```typescript
import {
hasAnalyticsConsent,
setAnalyticsConsent,
getAnalyticsConsentStatus
import {
hasAnalyticsConsent,
setAnalyticsConsent,
getAnalyticsConsentStatus,
} from '../lib/analytics';
// Check consent
if (hasAnalyticsConsent()) {
// Track event
// Track event
}
// Grant consent
@@ -197,6 +220,7 @@ const status = getAnalyticsConsentStatus(); // 'granted' | 'denied' | 'pending'
```
#### React Hooks
Location: `frontend/src/hooks/useAnalytics.ts`
```typescript
@@ -211,27 +235,30 @@ function App() {
// Manual tracking
function MyComponent() {
const { trackButtonClick, trackFeatureUsage, hasConsent } = useAnalytics();
const handleExport = () => {
trackButtonClick('export');
// ... export logic
};
return <button onClick={handleExport}>Export</button>;
}
```
#### Consent Banner
Location: `frontend/src/components/AnalyticsConsentBanner.tsx`
Shows automatically when user hasn't made a consent choice. Displays at bottom of page with Accept/Decline buttons.
#### Metrics Dashboard
Location: `frontend/src/pages/MetricsDashboard.tsx`
Access at: `/metrics`
**Features:**
- Summary cards (total events, unique users, consented users, avg response time)
- Top events bar chart
- Feature usage pie chart
@@ -243,33 +270,38 @@ Access at: `/metrics`
### Backend Tracking
#### Automatic Tracking
All API requests are automatically tracked via middleware. No additional code needed.
#### Manual Event Tracking
In route handlers:
```typescript
import { trackCustomEvent } from '../middleware/analyticsTracking';
router.post('/export', (req, res) => {
trackCustomEvent(req, 'data_export', { format: 'csv', rows: 100 });
// ... handle export
trackCustomEvent(req, 'data_export', { format: 'csv', rows: 100 });
// ... handle export
});
```
#### Feature Usage Tracking
```typescript
import { trackFeatureUsage } from '../services/analytics';
// Track when user uses a feature
await trackFeatureUsage({
featureName: 'ghost_analysis',
userId: user.id,
metadata: { resultCount: 25 },
consentGiven: user.analyticsConsent,
featureName: 'ghost_analysis',
userId: user.id,
metadata: { resultCount: 25 },
consentGiven: user.analyticsConsent,
});
```
#### Performance Tracking
```typescript
import { trackPerformance, PerformanceMetricType } from '../services/analytics';
@@ -278,20 +310,22 @@ const startTime = Date.now();
const duration = Date.now() - startTime;
await trackPerformance({
metricType: PerformanceMetricType.DB_QUERY,
metricName: 'fetch_ghost_scores',
value: duration,
unit: 'ms',
metadata: { rowCount: 100 },
metricType: PerformanceMetricType.DB_QUERY,
metricName: 'fetch_ghost_scores',
value: duration,
unit: 'ms',
metadata: { rowCount: 100 },
});
```
### Frontend Tracking
#### Page Views
Automatic via `usePageTracking()` hook in App component.
#### Button Clicks
```typescript
import { trackButtonClick } from '../lib/analytics';
@@ -304,22 +338,24 @@ import { trackButtonClick } from '../lib/analytics';
```
#### Feature Usage
```typescript
import { trackFeatureUsage } from '../lib/analytics';
const handleAnalysisRun = () => {
trackFeatureUsage('lurker_detection', { threshold: 5 });
// ... run analysis
trackFeatureUsage('lurker_detection', { threshold: 5 });
// ... run analysis
};
```
#### Form Submissions
```typescript
import { trackFormSubmit } from '../lib/analytics';
const handleSubmit = (data) => {
trackFormSubmit('user_settings', { fields: Object.keys(data) });
// ... submit form
trackFormSubmit('user_settings', { fields: Object.keys(data) });
// ... submit form
};
```
@@ -328,6 +364,7 @@ const handleSubmit = (data) => {
### Environment Variables
**Backend:**
```env
# Optional: Enable/disable analytics tracking
ENABLE_ANALYTICS=true
@@ -337,6 +374,7 @@ ANALYTICS_RETENTION_DAYS=90
```
**Frontend:**
```env
# Enable analytics tracking
VITE_ENABLE_ANALYTICS=true
@@ -348,6 +386,7 @@ VITE_ANALYTICS_TRACKING_ID=
### Data Retention
Configure retention policies in Prisma:
```typescript
// Example: Clean data older than 90 days
import { cleanOldAnalyticsData } from './services/analytics';
@@ -359,6 +398,7 @@ await cleanOldAnalyticsData(90);
### Daily Aggregation
Create scheduled jobs for daily summaries:
```typescript
import { aggregateDailySummary } from './services/analytics';
@@ -371,7 +411,9 @@ await aggregateDailySummary(yesterday);
## Privacy & GDPR Compliance
### Data Anonymization
When user consent is not given:
- User IDs are hashed (SHA-256, 16 chars)
- IP addresses are hashed
- Session IDs are hashed
@@ -379,19 +421,23 @@ When user consent is not given:
- `anonymized` flag set to `true`
### Consent Management
- Consent stored in localStorage and cookies
- Cookie synced to backend for server-side tracking
- Users can change consent at any time
- Declining consent anonymizes all future tracking
### Data Rights
Users have the right to:
- View their data (via existing privacy endpoints)
- Request deletion (via existing privacy endpoints)
- Opt-out of tracking (consent banner)
- Access summary of collected data
### Best Practices
1. Always check consent before tracking sensitive actions
2. Use generic event names (avoid PII in event names)
3. Store minimal data in properties
@@ -401,18 +447,21 @@ Users have the right to:
## Testing
### Unit Tests
```bash
cd backend
npm test -- __tests__/unit/services/analytics.test.ts
```
### Integration Tests
```bash
cd backend
npm test -- __tests__/integration/routes/metricsAnalytics.test.ts
```
### Manual Testing
1. Open application in browser
2. Accept/decline consent banner
3. Navigate pages (check page tracking)
@@ -423,10 +472,11 @@ npm test -- __tests__/integration/routes/metricsAnalytics.test.ts
## Monitoring
### Check Data Collection
```sql
-- Recent events
SELECT * FROM "UserAnalyticsEvent"
ORDER BY "createdAt" DESC
SELECT * FROM "UserAnalyticsEvent"
ORDER BY "createdAt" DESC
LIMIT 10;
-- Feature usage
@@ -444,24 +494,28 @@ ORDER BY avg_value DESC;
```
### Dashboard Access
- Frontend: Navigate to `/metrics`
- Backend API: `GET /api/metrics/dashboard`
## Troubleshooting
### Events Not Being Tracked
1. Check consent status in browser console
2. Verify `VITE_ENABLE_ANALYTICS=true` in frontend
3. Check network tab for failed requests
4. Review browser console for errors
### Dashboard Shows No Data
1. Verify database contains analytics records
2. Check API endpoint permissions
3. Verify user is authenticated
4. Check date range filters
### Performance Issues
1. Add database indexes (already included in schema)
2. Implement data retention cleanup
3. Use aggregated summaries instead of raw events
@@ -482,6 +536,7 @@ ORDER BY avg_value DESC;
## Support
For issues or questions:
1. Check this documentation
2. Review test files for examples
3. Check server logs for errors

118
BACKUP.md
View File

@@ -48,19 +48,19 @@ Configuration is handled by the scheduled tasks system in `src/utils/scheduledTa
### Backup Types
1. **Full Backup** (`BACKUP_TYPE=FULL`)
- Complete database dump
- Compressed with gzip
- Optionally encrypted with GPG
- Stored in S3 and locally
- Complete database dump
- Compressed with gzip
- Optionally encrypted with GPG
- Stored in S3 and locally
2. **Incremental Backup** (`BACKUP_TYPE=INCREMENTAL`)
- WAL (Write-Ahead Log) segments
- Enables point-in-time recovery
- Automatically archived every hour
- WAL (Write-Ahead Log) segments
- Enables point-in-time recovery
- Automatically archived every hour
3. **WAL Archive** (`BACKUP_TYPE=WAL_ARCHIVE`)
- Continuous archiving of transaction logs
- Required for point-in-time recovery
- Continuous archiving of transaction logs
- Required for point-in-time recovery
## Recovery Operations
@@ -98,6 +98,7 @@ cd scripts
```
Example:
```bash
./restore.sh s3://spywatcher-backups/postgres/full/backup.dump.gz "2024-01-25 14:30:00"
```
@@ -112,6 +113,7 @@ npm run backup:health-check
```
Returns:
- Last successful backup time
- Any issues detected
- Overall health status
@@ -124,6 +126,7 @@ npm run backup:stats
```
Returns:
- Total backups
- Success rate
- Average size and duration
@@ -143,8 +146,8 @@ Lists the 10 most recent backups with their status.
All backup operations are logged to the database in the `BackupLog` table:
```sql
SELECT * FROM "BackupLog"
ORDER BY "startedAt" DESC
SELECT * FROM "BackupLog"
ORDER BY "startedAt" DESC
LIMIT 10;
```
@@ -188,6 +191,7 @@ sudo ./setup-wal-archiving.sh
```
This will:
1. Configure PostgreSQL for WAL archiving
2. Set up archive command
3. Enable point-in-time recovery
@@ -201,6 +205,7 @@ sudo -u postgres psql -c "SELECT * FROM pg_stat_archiver;"
```
Check for:
- `archived_count` increasing over time
- `failed_count` should be 0
- `last_archived_time` should be recent
@@ -210,52 +215,57 @@ Check for:
### Backup Fails
**Check logs:**
```bash
tail -f /var/log/postgresql/postgresql-15-main.log
```
**Common issues:**
1. **Disk space full**
```bash
df -h /var/backups/spywatcher
```
```bash
df -h /var/backups/spywatcher
```
2. **Database connection issues**
```bash
psql -h $DB_HOST -U $DB_USER -d $DB_NAME -c "SELECT 1;"
```
```bash
psql -h $DB_HOST -U $DB_USER -d $DB_NAME -c "SELECT 1;"
```
3. **S3 permissions**
```bash
aws s3 ls s3://$S3_BUCKET/
```
```bash
aws s3 ls s3://$S3_BUCKET/
```
### Restore Fails
**Common issues:**
1. **File not found**
- Check backup file path
- Verify S3 bucket and key
- Ensure AWS credentials are configured
- Check backup file path
- Verify S3 bucket and key
- Ensure AWS credentials are configured
2. **Decryption fails**
- Verify GPG key is available
- Check GPG recipient matches
- Verify GPG key is available
- Check GPG recipient matches
3. **Database locked**
- Stop the application first
- Kill existing connections:
```sql
SELECT pg_terminate_backend(pg_stat_activity.pid)
FROM pg_stat_activity
WHERE pg_stat_activity.datname = 'spywatcher'
AND pid <> pg_backend_pid();
```
- Stop the application first
- Kill existing connections:
```sql
SELECT pg_terminate_backend(pg_stat_activity.pid)
FROM pg_stat_activity
WHERE pg_stat_activity.datname = 'spywatcher'
AND pid <> pg_backend_pid();
```
### No Recent Backups
**Check scheduled tasks:**
```bash
# Check if scheduled tasks are running
ps aux | grep node | grep scheduledTasks
@@ -265,6 +275,7 @@ tail -f logs/app.log
```
**Manual trigger:**
```bash
cd backend
npm run db:backup
@@ -273,13 +284,15 @@ npm run db:backup
### Backup Size Abnormal
**Check database size:**
```sql
SELECT pg_size_pretty(pg_database_size('spywatcher'));
```
**Check for data growth:**
```sql
SELECT
SELECT
schemaname,
tablename,
pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size
@@ -291,16 +304,19 @@ ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;
### WAL Archiving Not Working
**Check archive status:**
```sql
SELECT * FROM pg_stat_archiver;
```
**Check PostgreSQL config:**
```bash
grep -E "wal_level|archive_mode|archive_command" /etc/postgresql/15/main/postgresql.conf
```
**Check archive directory permissions:**
```bash
ls -la /var/lib/postgresql/wal_archive/
# or for S3
@@ -310,31 +326,31 @@ aws s3 ls s3://$S3_BUCKET/wal/
## Best Practices
1. **Test Restores Regularly**
- Monthly restore to test database
- Quarterly disaster recovery drills
- Document restore times
- Monthly restore to test database
- Quarterly disaster recovery drills
- Document restore times
2. **Monitor Backup Health**
- Review backup health check daily
- Set up alerts for failures
- Monitor backup size trends
- Review backup health check daily
- Set up alerts for failures
- Monitor backup size trends
3. **Keep Multiple Copies**
- Local backups (7 days)
- Primary S3 bucket (30 days)
- Secondary S3 bucket in different region
- Local backups (7 days)
- Primary S3 bucket (30 days)
- Secondary S3 bucket in different region
4. **Secure Your Backups**
- Enable encryption for sensitive data
- Use strong GPG keys
- Rotate keys regularly
- Restrict S3 bucket access
- Enable encryption for sensitive data
- Use strong GPG keys
- Rotate keys regularly
- Restrict S3 bucket access
5. **Document Everything**
- Keep this guide updated
- Document any custom procedures
- Maintain contact lists
- Record drill results
- Keep this guide updated
- Document any custom procedures
- Maintain contact lists
- Record drill results
## Emergency Contacts

133
CODE_OF_CONDUCT.md Normal file
View File

@@ -0,0 +1,133 @@
# Contributor Covenant Code of Conduct
## Our Pledge
We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, caste, color, religion, or sexual
identity and orientation.
We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.
## Our Standards
Examples of behavior that contributes to a positive environment for our
community include:
- Demonstrating empathy and kindness toward other people
- Being respectful of differing opinions, viewpoints, and experiences
- Giving and gracefully accepting constructive feedback
- Accepting responsibility and apologizing to those affected by our mistakes,
and learning from the experience
- Focusing on what is best not just for us as individuals, but for the overall
community
Examples of unacceptable behavior include:
- The use of sexualized language or imagery, and sexual attention or advances of
any kind
- Trolling, insulting or derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or email address,
without their explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Enforcement Responsibilities
Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.
Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.
## Scope
This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement through GitHub
issues or by contacting the project maintainers directly.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
reporter of any incident.
## Enforcement Guidelines
Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:
### 1. Correction
**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.
**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.
### 2. Warning
**Community Impact**: A violation through a single incident or series of
actions.
**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or permanent
ban.
### 3. Temporary Ban
**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.
**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.
### 4. Permanent Ban
**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior, harassment of an
individual, or aggression toward or disparagement of classes of individuals.
**Consequence**: A permanent ban from any sort of public interaction within the
community.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.1, available at
[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
Community Impact Guidelines were inspired by
[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
For answers to common questions about this code of conduct, see the FAQ at
[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
[https://www.contributor-covenant.org/translations][translations].
[homepage]: https://www.contributor-covenant.org
[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[Mozilla CoC]: https://github.com/mozilla/diversity
[FAQ]: https://www.contributor-covenant.org/faq
[translations]: https://www.contributor-covenant.org/translations

369
CONNECTION_POOLING.md
View File

@@ -82,13 +82,14 @@ server_reset_query = DISCARD ALL
### Pool Modes Explained
| Mode | Description | Use Case |
|------|-------------|----------|
| **session** | One server connection per client | Long-running sessions, advisory locks |
| Mode | Description | Use Case |
| --------------- | ------------------------------------- | ------------------------------------------ |
| **session** | One server connection per client | Long-running sessions, advisory locks |
| **transaction** | One server connection per transaction | Most applications (recommended for Prisma) |
| **statement** | One server connection per statement | Stateless applications only |
| **statement** | One server connection per statement | Stateless applications only |
**We use `transaction` mode** because:
- Compatible with Prisma's transaction handling
- Efficient connection reuse
- Balances performance and compatibility
@@ -99,26 +100,26 @@ server_reset_query = DISCARD ALL
```yaml
pgbouncer:
build:
context: ./pgbouncer
environment:
DB_USER: spywatcher
DB_PASSWORD: ${DB_PASSWORD}
ports:
- "6432:6432"
build:
context: ./pgbouncer
environment:
DB_USER: spywatcher
DB_PASSWORD: ${DB_PASSWORD}
ports:
- '6432:6432'
```
#### Production
```yaml
pgbouncer:
build:
context: ./pgbouncer
environment:
DB_USER: spywatcher
DB_PASSWORD: ${DB_PASSWORD}
restart: unless-stopped
# Note: No external port exposure in production
build:
context: ./pgbouncer
environment:
DB_USER: spywatcher
DB_PASSWORD: ${DB_PASSWORD}
restart: unless-stopped
# Note: No external port exposure in production
```
### Environment Variables
@@ -142,27 +143,31 @@ When using PgBouncer, Prisma needs fewer connections:
```typescript
const db = new PrismaClient({
datasources: {
db: {
url: process.env.DATABASE_URL,
datasources: {
db: {
url: process.env.DATABASE_URL,
},
},
},
});
```
### Connection URL Parameters
#### With PgBouncer (Production)
```
postgresql://user:password@pgbouncer:6432/dbname?pgbouncer=true
```
- Keep connection pool small (Prisma default: 5)
- PgBouncer handles the actual pooling
#### Direct Connection (Development/Migrations)
```
postgresql://user:password@postgres:5432/dbname?connection_limit=20&pool_timeout=20
```
- `connection_limit`: 10-50 depending on load
- `pool_timeout`: 20 seconds
- `connect_timeout`: 10 seconds
@@ -170,16 +175,19 @@ postgresql://user:password@postgres:5432/dbname?connection_limit=20&pool_timeout
### Why Fewer Connections with PgBouncer?
Without PgBouncer:
```
Application → PostgreSQL (need many connections)
```
With PgBouncer:
```
Application → PgBouncer → PostgreSQL (PgBouncer reuses connections)
```
Example with 10 application instances:
- **Without PgBouncer**: 10 × 20 = 200 PostgreSQL connections needed
- **With PgBouncer**: 10 × 5 = 50 client connections → 25 PostgreSQL connections
@@ -188,20 +196,22 @@ Example with 10 application instances:
### Application Startup
1. **Database Connection**
```typescript
// db.ts initializes Prisma Client
export const db = new PrismaClient({ ... });
```
```typescript
// db.ts initializes Prisma Client
export const db = new PrismaClient({ ... });
```
2. **Redis Connection** (if enabled)
```typescript
// redis.ts initializes Redis client
const redisClient = new Redis(url, { ... });
```
```typescript
// redis.ts initializes Redis client
const redisClient = new Redis(url, { ... });
```
3. **Health Checks**
- Database connectivity verification
- Connection pool metrics collection
- Database connectivity verification
- Connection pool metrics collection
### During Operation
@@ -214,14 +224,14 @@ Example with 10 application instances:
```typescript
// Signal handlers in db.ts and redis.ts
process.on('SIGTERM', async () => {
// 1. Stop accepting new connections
// 2. Wait for in-flight requests
// 3. Close Prisma connections
await db.$disconnect();
// 4. Close Redis connections
await closeRedisConnection();
// 5. Exit process
process.exit(0);
// 1. Stop accepting new connections
// 2. Wait for in-flight requests
// 3. Close Prisma connections
await db.$disconnect();
// 4. Close Redis connections
await closeRedisConnection();
// 5. Exit process
process.exit(0);
});
```
@@ -245,70 +255,74 @@ process.on('SIGTERM', async () => {
### Health Check Endpoints
#### System Health
```bash
GET /api/admin/monitoring/connections/health
```
Returns:
```json
{
"healthy": true,
"timestamp": "2025-01-15T10:30:00Z",
"database": {
"healthy": true,
"responseTime": 12,
"connectionPool": {
"active": 3,
"idle": 2,
"total": 5,
"max": 100,
"utilizationPercent": "5.00",
"isPgBouncer": true,
"isShuttingDown": false
"timestamp": "2025-01-15T10:30:00Z",
"database": {
"healthy": true,
"responseTime": 12,
"connectionPool": {
"active": 3,
"idle": 2,
"total": 5,
"max": 100,
"utilizationPercent": "5.00",
"isPgBouncer": true,
"isShuttingDown": false
}
},
"redis": {
"available": true,
"connected": true,
"status": "ready"
}
},
"redis": {
"available": true,
"connected": true,
"status": "ready"
}
}
```
#### Connection Pool Stats
```bash
GET /api/admin/monitoring/connections/pool
```
Returns:
```json
{
"database": {
"utilizationPercent": 5.0,
"activeConnections": 3,
"maxConnections": 100,
"isHealthy": true
},
"redis": {
"available": true,
"connected": true
}
"database": {
"utilizationPercent": 5.0,
"activeConnections": 3,
"maxConnections": 100,
"isHealthy": true
},
"redis": {
"available": true,
"connected": true
}
}
```
#### Connection Alerts
```bash
GET /api/admin/monitoring/connections/alerts
```
Returns:
```json
{
"alerts": [
"WARNING: Database connection pool at 85% utilization"
],
"count": 1,
"timestamp": "2025-01-15T10:30:00Z"
"alerts": ["WARNING: Database connection pool at 85% utilization"],
"count": 1,
"timestamp": "2025-01-15T10:30:00Z"
}
```
@@ -379,6 +393,7 @@ Metrics:
### Issue: Too many connections
**Symptoms:**
```
Error: remaining connection slots are reserved for non-replication superuser connections
```
@@ -386,25 +401,28 @@ Error: remaining connection slots are reserved for non-replication superuser con
**Solutions:**
1. **Check PgBouncer pool size:**
```bash
# In pgbouncer.ini
default_pool_size = 25 # Increase if needed
max_db_connections = 50
```
```bash
# In pgbouncer.ini
default_pool_size = 25 # Increase if needed
max_db_connections = 50
```
2. **Check PostgreSQL max_connections:**
```sql
SHOW max_connections; -- Should be > PgBouncer pool size
```
```sql
SHOW max_connections; -- Should be > PgBouncer pool size
```
3. **Monitor connection usage:**
```bash
curl http://localhost:3001/api/admin/monitoring/connections/pool
```
```bash
curl http://localhost:3001/api/admin/monitoring/connections/pool
```
### Issue: Connection timeouts
**Symptoms:**
```
Error: Connection timeout
```
@@ -412,50 +430,56 @@ Error: Connection timeout
**Solutions:**
1. **Check PgBouncer is running:**
```bash
docker ps | grep pgbouncer
```
```bash
docker ps | grep pgbouncer
```
2. **Check connection string:**
```bash
# Ensure using correct host and port
DATABASE_URL=postgresql://user:pass@pgbouncer:6432/db?pgbouncer=true
```
```bash
# Ensure using correct host and port
DATABASE_URL=postgresql://user:pass@pgbouncer:6432/db?pgbouncer=true
```
3. **Increase timeouts:**
```ini
# In pgbouncer.ini
query_wait_timeout = 120
server_connect_timeout = 15
```
```ini
# In pgbouncer.ini
query_wait_timeout = 120
server_connect_timeout = 15
```
### Issue: Slow queries with PgBouncer
**Symptoms:**
- Queries slower than without PgBouncer
**Solutions:**
1. **Ensure using transaction mode:**
```ini
pool_mode = transaction # Not session mode
```
```ini
pool_mode = transaction # Not session mode
```
2. **Check for connection reuse:**
```sql
-- In PgBouncer admin
SHOW POOLS;
-- Check cl_active, cl_waiting, sv_active, sv_idle
```
```sql
-- In PgBouncer admin
SHOW POOLS;
-- Check cl_active, cl_waiting, sv_active, sv_idle
```
3. **Monitor query wait time:**
```bash
curl http://localhost:3001/api/admin/monitoring/database/slow-queries
```
```bash
curl http://localhost:3001/api/admin/monitoring/database/slow-queries
```
### Issue: Migrations fail with PgBouncer
**Symptoms:**
```
Error: prepared statement already exists
```
@@ -463,120 +487,128 @@ Error: prepared statement already exists
**Solution:**
Always run migrations with direct PostgreSQL connection:
```bash
# Use DATABASE_URL_DIRECT for migrations
DATABASE_URL=$DATABASE_URL_DIRECT npx prisma migrate deploy
```
Or configure in docker-compose.yml:
```yaml
migrate:
environment:
DATABASE_URL: postgresql://user:pass@postgres:5432/db # Direct connection
environment:
DATABASE_URL: postgresql://user:pass@postgres:5432/db # Direct connection
```
### Issue: Connection pool exhaustion
**Symptoms:**
- "Pool is full" errors
- High connection utilization
**Solutions:**
1. **Scale PgBouncer pool:**
```ini
default_pool_size = 50 # Increase from 25
reserve_pool_size = 10 # Increase reserve
```
```ini
default_pool_size = 50 # Increase from 25
reserve_pool_size = 10 # Increase reserve
```
2. **Add connection cleanup:**
```typescript
// Ensure proper $disconnect() on errors
try {
await db.query();
} finally {
// Connections released automatically
}
```
```typescript
// Ensure proper $disconnect() on errors
try {
await db.query();
} finally {
// Connections released automatically
}
```
3. **Reduce connection limit per instance:**
```
# Fewer connections per app instance
DATABASE_URL=...?connection_limit=3
```
```
# Fewer connections per app instance
DATABASE_URL=...?connection_limit=3
```
## ✅ Best Practices
### Production Deployment
1. **Always use PgBouncer in production**
- Better connection management
- Prevents connection exhaustion
- Enables horizontal scaling
- Better connection management
- Prevents connection exhaustion
- Enables horizontal scaling
2. **Configure appropriate pool sizes**
```
PgBouncer pool: 25-50 connections
Prisma per instance: 3-5 connections
PostgreSQL max: 100+ connections
```
```
PgBouncer pool: 25-50 connections
Prisma per instance: 3-5 connections
PostgreSQL max: 100+ connections
```
3. **Use separate connections for migrations**
- Migrations need direct PostgreSQL access
- Bypass PgBouncer for schema changes
- Migrations need direct PostgreSQL access
- Bypass PgBouncer for schema changes
4. **Monitor connection metrics**
- Set up alerts for >80% utilization
- Track connection pool trends
- Monitor slow query counts
- Set up alerts for >80% utilization
- Track connection pool trends
- Monitor slow query counts
### Development Practices
1. **Test with and without PgBouncer**
- Dev: direct connection (easier debugging)
- Staging/Prod: through PgBouncer
- Dev: direct connection (easier debugging)
- Staging/Prod: through PgBouncer
2. **Use environment-specific configs**
```bash
# .env.development
DATABASE_URL=postgresql://...@postgres:5432/db
# .env.production
DATABASE_URL=postgresql://...@pgbouncer:6432/db?pgbouncer=true
```
```bash
# .env.development
DATABASE_URL=postgresql://...@postgres:5432/db
# .env.production
DATABASE_URL=postgresql://...@pgbouncer:6432/db?pgbouncer=true
```
3. **Implement proper error handling**
```typescript
try {
await db.query();
} catch (error) {
// Log error
// Connection automatically released
throw error;
}
```
```typescript
try {
await db.query();
} catch (error) {
// Log error
// Connection automatically released
throw error;
}
```
4. **Use connection pooling metrics**
- Monitor during load tests
- Adjust pool sizes based on metrics
- Set up automated alerts
- Monitor during load tests
- Adjust pool sizes based on metrics
- Set up automated alerts
### Security Considerations
1. **Secure PgBouncer credentials**
- Use strong passwords
- Rotate credentials regularly
- Use environment variables
- Use strong passwords
- Rotate credentials regularly
- Use environment variables
2. **Limit PgBouncer access**
- Don't expose port externally
- Use internal Docker network
- Configure firewall rules
- Don't expose port externally
- Use internal Docker network
- Configure firewall rules
3. **Monitor for connection abuse**
- Track connection patterns
- Alert on unusual spikes
- Implement rate limiting
- Track connection patterns
- Alert on unusual spikes
- Implement rate limiting
## 📚 Additional Resources
@@ -589,6 +621,7 @@ migrate:
## 🆘 Support
For issues or questions:
- Check monitoring endpoints first
- Review logs for error messages
- Consult troubleshooting section

348
CONTRIBUTING.md
View File

@@ -1,63 +1,299 @@
# Contributing to Discord Spywatcher
Thank you for your interest in contributing to Discord Spywatcher! This document provides guidelines and instructions for contributing to the project.
Thank you for your interest in contributing to Discord Spywatcher! 🎉
We're excited to have you here and grateful for your contributions, whether it's reporting bugs, proposing features, improving documentation, or writing code. This guide will help you get started.
## Table of Contents
- [Code of Conduct](#code-of-conduct)
- [Getting Started](#getting-started)
- [Ways to Contribute](#ways-to-contribute)
- [Development Setup](#development-setup)
- [Development Workflow](#development-workflow)
- [Code Quality Standards](#code-quality-standards)
- [Commit Guidelines](#commit-guidelines)
- [Pull Request Process](#pull-request-process)
- [Issue Guidelines](#issue-guidelines)
## Code of Conduct
Please be respectful and professional in all interactions with other contributors.
This project and everyone participating in it is governed by our [Code of Conduct](./CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to the project maintainers through GitHub issues or direct contact.
## Getting Started
1. Fork the repository
2. Clone your fork: `git clone https://github.com/YOUR_USERNAME/discord-spywatcher.git`
3. Create a new branch: `git checkout -b feature/your-feature-name`
4. Make your changes
5. Push to your fork and submit a pull request
### First Time Contributors
If this is your first time contributing to open source, welcome! Here are some resources to help you get started:
- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
- [First Contributions](https://github.com/firstcontributions/first-contributions)
- [GitHub Flow](https://guides.github.com/introduction/flow/)
### Quick Start Guide
1. **Fork the repository** - Click the "Fork" button at the top right of the repository page
2. **Clone your fork**:
```bash
git clone https://github.com/YOUR_USERNAME/discord-spywatcher.git
cd discord-spywatcher
```
3. **Add upstream remote**:
```bash
git remote add upstream https://github.com/subculture-collective/discord-spywatcher.git
```
4. **Create a new branch**:
```bash
git checkout -b feature/your-feature-name
# or
git checkout -b fix/your-bug-fix
```
5. **Make your changes** - Follow the development setup and guidelines below
6. **Push to your fork**:
```bash
git push origin feature/your-feature-name
```
7. **Submit a pull request** - Go to your fork on GitHub and click "New Pull Request"
## Ways to Contribute
There are many ways to contribute to Discord Spywatcher:
### 🐛 Report Bugs
Found a bug? Please [create a bug report](.github/ISSUE_TEMPLATE/bug_report.yml) with:
- Clear description of the issue
- Steps to reproduce
- Expected vs. actual behavior
- Your environment details
### 💡 Suggest Features
Have an idea for a new feature? [Submit a feature request](.github/ISSUE_TEMPLATE/feature_request.yml) with:
- Description of the problem you're trying to solve
- Your proposed solution
- Any alternative approaches you've considered
### 📝 Improve Documentation
Documentation improvements are always welcome:
- Fix typos or clarify existing docs
- Add examples and tutorials
- Improve code comments
- Write guides for new features
Use the [documentation template](.github/ISSUE_TEMPLATE/documentation.yml) to suggest improvements.
### 🔧 Write Code
Ready to contribute code? Great!
- Check the [issue tracker](https://github.com/subculture-collective/discord-spywatcher/issues) for open issues
- Look for issues labeled `good first issue` or `help wanted`
- Comment on an issue to let others know you're working on it
- Follow the development workflow below
### 🧪 Write Tests
Help improve code coverage:
- Add tests for existing features
- Improve test quality and coverage
- Add integration and end-to-end tests
### 👀 Review Pull Requests
Help review open pull requests:
- Test the changes locally
- Provide constructive feedback
- Check for code quality and best practices
## Development Setup
### Prerequisites
- Node.js (v18 or higher)
- npm or yarn
- Git
Before you begin, ensure you have the following installed:
- **Node.js** (v18 or higher) - [Download](https://nodejs.org/)
- **npm** (comes with Node.js) or **yarn**
- **Git** - [Download](https://git-scm.com/)
- **Docker** (optional but recommended) - [Download](https://www.docker.com/)
- **PostgreSQL** (if not using Docker) - [Download](https://www.postgresql.org/)
### Installation
1. Install dependencies for both backend and frontend:
#### Option 1: Using Docker (Recommended)
The easiest way to get started:
```bash
# Install root dependencies (for git hooks)
npm install
# Copy environment file and configure
cp .env.example .env
# Edit .env with your Discord credentials
# Install backend dependencies
cd backend
npm install
# Start development environment
docker-compose -f docker-compose.dev.yml up
```
# Install frontend dependencies
cd ../frontend
Access:
- Frontend: http://localhost:5173
- Backend API: http://localhost:3001
- PostgreSQL: localhost:5432
See [DOCKER.md](./DOCKER.md) for detailed Docker setup.
#### Option 2: Manual Setup
1. **Install root dependencies** (for git hooks and tooling):
```bash
npm install
```
2. Set up environment variables:
2. **Install backend dependencies**:
```bash
cd backend
npm install
```
3. **Install frontend dependencies**:
```bash
cd frontend
npm install
```
4. **Set up environment variables**:
```bash
# Backend
cp backend/backend.env.example backend/.env
cp backend/.env.example backend/.env
# Edit backend/.env with your configuration
# Frontend
cp frontend/frontend.env.example frontend/.env
cp frontend/.env.example frontend/.env
# Edit frontend/.env with your configuration
```
5. **Set up the database**:
```bash
cd backend
npx prisma migrate dev --name init
npx prisma generate
```
6. **Start the development servers**:
```bash
# In terminal 1 (Backend API)
cd backend
npm run dev:api
# In terminal 2 (Discord Bot)
cd backend
npm run dev
# In terminal 3 (Frontend)
cd frontend
npm run dev
```
### Discord Application Setup
To run Discord Spywatcher, you need to create a Discord application:
1. Go to the [Discord Developer Portal](https://discord.com/developers/applications)
2. Click "New Application" and give it a name
3. Navigate to the "Bot" section and click "Add Bot"
4. Under "Privileged Gateway Intents", enable:
- Presence Intent
- Server Members Intent
- Message Content Intent
5. Copy the bot token and add it to your `.env` file as `DISCORD_BOT_TOKEN`
6. Navigate to "OAuth2" → "General"
7. Copy the Client ID and Client Secret to your `.env` file
8. Add your redirect URI (e.g., `http://localhost:5173/auth/callback`)
9. Navigate to "OAuth2" → "URL Generator"
10. Select scopes: `bot`, `identify`, `guilds`
11. Select bot permissions: `View Channels`, `Read Message History`, `Send Messages`
12. Copy the generated URL and use it to invite the bot to your server
## Development Workflow
### Keeping Your Fork Updated
Before starting work on a new feature or fix, sync your fork with the upstream repository:
```bash
# Fetch upstream changes
git fetch upstream
# Switch to your main branch
git checkout main
# Merge upstream changes
git merge upstream/main
# Push to your fork
git push origin main
```
### Working on a Feature or Fix
1. **Create a feature branch** from the latest `main`:
```bash
git checkout main
git pull upstream main
git checkout -b feature/your-feature-name
```
2. **Make your changes** following the code standards
3. **Test your changes**:
```bash
# Run linting
npm run lint
# Run type checking
npm run type-check
# Run tests
cd backend && npm test
cd frontend && npm test
```
4. **Commit your changes** using conventional commits:
```bash
git add .
git commit -m "feat(component): add new feature"
```
5. **Push to your fork**:
```bash
git push origin feature/your-feature-name
```
6. **Create a Pull Request** on GitHub
### Development Tips
- **Use the git hooks**: They're set up automatically and will catch issues before you push
- **Run tests frequently**: Catch issues early
- **Keep commits small and focused**: Easier to review and revert if needed
- **Write clear commit messages**: Help others understand your changes
- **Update documentation**: Keep docs in sync with code changes
## Code Quality Standards
This project uses several tools to maintain code quality:
@@ -282,12 +518,80 @@ refactor(dashboard): extract user list component
- Explain the reasoning behind suggestions
- Approve when satisfied with changes
## Issue Guidelines
### Creating Good Issues
When creating an issue, please:
- **Use a clear and descriptive title**
- **Search for existing issues** to avoid duplicates
- **Use the appropriate template** (bug report, feature request, or documentation)
- **Provide complete information** - the more details, the better
- **Stay on topic** - keep discussions focused on the issue at hand
- **Be respectful** - follow our Code of Conduct
### Issue Labels
We use labels to categorize issues:
- `bug` - Something isn't working
- `enhancement` - New feature or request
- `documentation` - Improvements or additions to documentation
- `good first issue` - Good for newcomers
- `help wanted` - Extra attention is needed
- `needs-triage` - Needs to be reviewed by maintainers
- `priority: high` - High priority issue
- `priority: low` - Low priority issue
- `wontfix` - This will not be worked on
### Issue Lifecycle
1. **New** - Issue is created using a template
2. **Triage** - Maintainers review and label the issue
3. **Accepted** - Issue is confirmed and ready for work
4. **In Progress** - Someone is actively working on it
5. **Review** - Pull request is under review
6. **Done** - Issue is resolved and closed
## Community Guidelines
### Communication
- **Be kind and courteous** - We're all here to learn and help each other
- **Be patient** - Maintainers and contributors are often volunteers
- **Be constructive** - Focus on the issue, not the person
- **Be clear** - Explain your ideas thoroughly
- **Be respectful of time** - Keep discussions focused and productive
### Getting Help
Need help with something? Here are the best ways to get support:
- **Documentation** - Check the [README](./README.md) and [docs](./docs/) first
- **Discussions** - Use [GitHub Discussions](https://github.com/subculture-collective/discord-spywatcher/discussions) for questions
- **Issues** - Create an issue if you've found a bug or want to suggest a feature
- **Pull Request Comments** - Ask questions directly on relevant PRs
## Recognition
We value all contributions! Contributors are recognized in:
- GitHub's contributor graph
- Release notes for significant contributions
- The project's community
## Need Help?
- Check existing issues and discussions
- Check existing [issues](https://github.com/subculture-collective/discord-spywatcher/issues) and [discussions](https://github.com/subculture-collective/discord-spywatcher/discussions)
- Read the [documentation](./README.md)
- Ask questions in pull request comments
- Reach out to maintainers
## License
By contributing to Discord Spywatcher, you agree that your contributions will be licensed under the same license as the project.
---
Thank you for contributing to Discord Spywatcher! 🙏

52
DATABASE_OPTIMIZATION.md
View File

@@ -5,6 +5,7 @@ This document outlines the database optimization strategy for Discord Spywatcher
## 📊 Overview
The database optimization implementation focuses on:
- **Strategic indexing** for common query patterns
- **Query optimization** to reduce database load
- **Performance monitoring** to identify bottlenecks
@@ -17,28 +18,33 @@ The database optimization implementation focuses on:
Composite indexes are created for common query patterns that filter by multiple columns:
#### PresenceEvent
- `(userId, createdAt DESC)` - User presence history queries
- `(userId)` - Single user lookups
- `(createdAt)` - Time-based queries
#### MessageEvent
- `(userId, createdAt DESC)` - User message history
- `(guildId, channelId)` - Guild-channel message queries
- `(guildId, createdAt DESC)` - Guild message history
- Individual indexes on `userId`, `guildId`, `channelId`, `createdAt`
#### TypingEvent
- `(userId, channelId)` - User typing in specific channels
- `(guildId, createdAt DESC)` - Guild typing activity over time
- Individual indexes on `userId`, `guildId`, `channelId`, `createdAt`
#### ReactionTime
- `(observerId, createdAt DESC)` - Observer reaction history
- `(guildId, createdAt DESC)` - Guild reaction history
- `(deltaMs)` - Fast reaction queries
- Individual indexes on `observerId`, `actorId`, `guildId`, `createdAt`
#### User
- `(role)` - Role-based queries
- `(lastSeenAt DESC)` - Last seen queries
- `(role, lastSeenAt DESC)` - Combined role and activity queries
@@ -65,6 +71,7 @@ GIN (Generalized Inverted Index) indexes for JSONB columns enable efficient JSON
### Ghost Detection
**Before** (N+1 query pattern):
```typescript
// Multiple separate queries - inefficient
const typings = await db.typingEvent.groupBy(...);
@@ -73,6 +80,7 @@ const messages = await db.messageEvent.groupBy(...);
```
**After** (Single optimized query):
```typescript
// Single aggregation query using raw SQL
const result = await db.$queryRaw`
@@ -91,6 +99,7 @@ const result = await db.$queryRaw`
### Lurker Detection
Optimized from multiple `findMany` calls to a single query with subqueries:
- Identifies users with presence but no activity
- Uses LEFT JOIN to efficiently find users without matching activity records
- Filters in database rather than application code
@@ -98,6 +107,7 @@ Optimized from multiple `findMany` calls to a single query with subqueries:
### Reaction Stats
Changed from in-memory aggregation to database-level aggregation:
- Uses SQL `AVG()` and `COUNT() FILTER` for efficient calculation
- Reduces data transfer from database to application
- Handles filtering at database level
@@ -110,11 +120,12 @@ The application includes a Prisma middleware that tracks slow queries:
```typescript
// Configurable thresholds (env variables)
SLOW_QUERY_THRESHOLD_MS=100 // Warn threshold
CRITICAL_QUERY_THRESHOLD_MS=1000 // Critical threshold
SLOW_QUERY_THRESHOLD_MS = 100; // Warn threshold
CRITICAL_QUERY_THRESHOLD_MS = 1000; // Critical threshold
```
Features:
- Logs queries exceeding thresholds to console
- Stores last 100 slow queries in memory
- Provides statistics API for monitoring dashboards
@@ -146,17 +157,20 @@ The `databaseMaintenance.ts` utility provides:
### Initial Setup
1. Apply Prisma migrations:
```bash
cd backend
npm run prisma:migrate
```
2. Apply PostgreSQL-specific indexes:
```bash
psql -d spywatcher -f ../scripts/add-performance-indexes.sql
```
3. Initialize full-text search (if not already done):
```bash
npm run db:fulltext
```
@@ -164,30 +178,36 @@ npm run db:fulltext
### Regular Maintenance
#### Weekly
- Review slow query logs via monitoring dashboard
- Check index usage statistics
- Review table growth trends
#### Monthly
- Run ANALYZE on all tables:
```bash
curl -X POST http://localhost:3000/api/admin/monitoring/database/analyze \
-H "Authorization: Bearer YOUR_TOKEN"
```
- Check for unused indexes:
```bash
curl http://localhost:3000/api/admin/monitoring/database/indexes \
-H "Authorization: Bearer YOUR_TOKEN"
```
- Review maintenance report:
```bash
curl http://localhost:3000/api/admin/monitoring/database/report \
-H "Authorization: Bearer YOUR_TOKEN"
```
#### Quarterly
- Review and remove truly unused indexes
- Consider table partitioning for very large tables
- Review and adjust connection pool settings
@@ -197,7 +217,7 @@ curl http://localhost:3000/api/admin/monitoring/database/report \
Check for index bloat periodically:
```sql
SELECT
SELECT
schemaname, tablename, indexname,
pg_size_pretty(pg_relation_size(indexrelid)) as index_size
FROM pg_stat_user_indexes
@@ -206,6 +226,7 @@ ORDER BY pg_relation_size(indexrelid) DESC;
```
Rebuild bloated indexes:
```sql
REINDEX INDEX CONCURRENTLY idx_name;
```
@@ -224,6 +245,7 @@ Based on the issue requirements:
## 🔍 Monitoring Best Practices
1. **Enable pg_stat_statements** for PostgreSQL query tracking:
```sql
-- Add to postgresql.conf
shared_preload_libraries = 'pg_stat_statements'
@@ -231,34 +253,37 @@ pg_stat_statements.track = all
```
2. **Set up alerts** for:
- Queries exceeding 1000ms
- Index usage below 50% on tables > 10k rows
- Connection pool saturation (> 80% usage)
- Table sizes growing abnormally
- Queries exceeding 1000ms
- Index usage below 50% on tables > 10k rows
- Connection pool saturation (> 80% usage)
- Table sizes growing abnormally
3. **Regular reviews** of:
- Slow query patterns
- Index hit ratios
- Cache effectiveness
- Connection pool metrics
- Slow query patterns
- Index hit ratios
- Cache effectiveness
- Connection pool metrics
## 🛠️ Troubleshooting
### Query Performance Issues
1. Check EXPLAIN ANALYZE output:
```sql
EXPLAIN ANALYZE SELECT * FROM "MessageEvent"
EXPLAIN ANALYZE SELECT * FROM "MessageEvent"
WHERE "guildId" = 'xxx' AND "createdAt" > NOW() - INTERVAL '7 days';
```
2. Verify index usage:
```sql
SELECT * FROM pg_stat_user_indexes
SELECT * FROM pg_stat_user_indexes
WHERE tablename = 'MessageEvent';
```
3. Check for sequential scans on large tables:
```sql
SELECT schemaname, tablename, seq_scan, seq_tup_read, idx_scan
FROM pg_stat_user_tables
@@ -276,6 +301,7 @@ ORDER BY seq_scan DESC;
### Index Not Being Used
Common reasons:
1. Statistics are outdated - Run ANALYZE
2. Small table size - PostgreSQL may prefer sequential scan
3. Poor selectivity - Index doesn't filter enough rows

543
DISASTER_RECOVERY.md
View File

@@ -6,16 +6,17 @@ This document provides detailed procedures for recovering from various disaster
## Quick Reference
| Scenario | RTO | RPO | Primary Contact |
|----------|-----|-----|----------------|
| Database Corruption | 2 hours | 1 hour | Database Admin |
| Complete Infrastructure Failure | 4 hours | 1 hour | DevOps Lead |
| Regional Outage | 6 hours | 1 hour | Cloud Architect |
| Ransomware Attack | 3 hours | 1 hour | Security Team |
| Scenario | RTO | RPO | Primary Contact |
| ------------------------------- | ------- | ------ | --------------- |
| Database Corruption | 2 hours | 1 hour | Database Admin |
| Complete Infrastructure Failure | 4 hours | 1 hour | DevOps Lead |
| Regional Outage | 6 hours | 1 hour | Cloud Architect |
| Ransomware Attack | 3 hours | 1 hour | Security Team |
## Prerequisites
### Required Access
- [ ] Database credentials (DB_PASSWORD)
- [ ] AWS CLI configured with appropriate permissions
- [ ] S3 bucket access (spywatcher-backups)
@@ -24,6 +25,7 @@ This document provides detailed procedures for recovering from various disaster
- [ ] Admin access to cloud provider console
### Required Tools
- [ ] PostgreSQL client tools (psql, pg_restore)
- [ ] AWS CLI
- [ ] GPG/OpenSSL
@@ -37,21 +39,21 @@ This document provides detailed procedures for recovering from various disaster
Our backup strategy includes:
1. **Full Database Backups** (Daily at 2 AM UTC)
- Compressed with gzip
- Encrypted with GPG
- Stored in primary and secondary S3 buckets
- Retention: 30 days daily, 12 months (monthly snapshots)
- Compressed with gzip
- Encrypted with GPG
- Stored in primary and secondary S3 buckets
- Retention: 30 days daily, 12 months (monthly snapshots)
2. **Incremental Backups** (Every 6 hours)
- WAL archiving for point-in-time recovery
- Stored in S3
- Retention: 7 days
- WAL archiving for point-in-time recovery
- Stored in S3
- Retention: 7 days
3. **Configuration Backups** (On change)
- Environment variables
- SSL certificates
- Application configuration files
- Infrastructure as Code (Terraform/CloudFormation)
- Environment variables
- SSL certificates
- Application configuration files
- Infrastructure as Code (Terraform/CloudFormation)
### Backup Locations
@@ -65,6 +67,7 @@ Our backup strategy includes:
### Scenario 1: Database Corruption
**Symptoms:**
- Data inconsistencies
- Query errors
- Failed integrity checks
@@ -73,107 +76,115 @@ Our backup strategy includes:
**Recovery Steps:**
1. **Assess the Damage** (10 minutes)
```bash
# Connect to database
psql -h $DB_HOST -U spywatcher -d spywatcher
# Check for errors in logs
tail -100 /var/log/postgresql/postgresql-15-main.log
# Run integrity checks
SELECT * FROM pg_stat_database WHERE datname = 'spywatcher';
```
```bash
# Connect to database
psql -h $DB_HOST -U spywatcher -d spywatcher
# Check for errors in logs
tail -100 /var/log/postgresql/postgresql-15-main.log
# Run integrity checks
SELECT * FROM pg_stat_database WHERE datname = 'spywatcher';
```
2. **Stop the Application** (5 minutes)
```bash
# If using Kubernetes
kubectl scale deployment spywatcher-backend --replicas=0
# If using Docker Compose
docker-compose stop backend
# If using systemd
sudo systemctl stop spywatcher-backend
```
```bash
# If using Kubernetes
kubectl scale deployment spywatcher-backend --replicas=0
# If using Docker Compose
docker-compose stop backend
# If using systemd
sudo systemctl stop spywatcher-backend
```
3. **Identify Last Known Good Backup** (5 minutes)
```bash
# List recent backups
aws s3 ls s3://spywatcher-backups/postgres/full/ --recursive | sort -r | head -10
# Check backup logs
cd $PROJECT_ROOT/backend
npm run db:backup-logs
```
```bash
# List recent backups
aws s3 ls s3://spywatcher-backups/postgres/full/ --recursive | sort -r | head -10
# Check backup logs
cd $PROJECT_ROOT/backend
npm run db:backup-logs
```
4. **Restore Database** (60 minutes)
```bash
# Download and restore the backup
cd $PROJECT_ROOT/scripts
# Set environment variables
export DB_NAME="spywatcher"
export DB_USER="spywatcher"
export DB_PASSWORD="your_password"
export DB_HOST="localhost"
export S3_BUCKET="spywatcher-backups"
# Run restore
./restore.sh s3://spywatcher-backups/postgres/full/spywatcher_full_20240125_120000.dump.gz
```
```bash
# Download and restore the backup
cd $PROJECT_ROOT/scripts
# Set environment variables
export DB_NAME="spywatcher"
export DB_USER="spywatcher"
export DB_PASSWORD="your_password"
export DB_HOST="localhost"
export S3_BUCKET="spywatcher-backups"
# Run restore
./restore.sh s3://spywatcher-backups/postgres/full/spywatcher_full_20240125_120000.dump.gz
```
5. **Verify Data Integrity** (15 minutes)
```bash
# Run data integrity checks
psql -h $DB_HOST -U spywatcher -d spywatcher -c "
SELECT
(SELECT COUNT(*) FROM \"User\") as users,
(SELECT COUNT(*) FROM \"Guild\") as guilds,
(SELECT COUNT(*) FROM \"ApiKey\") as api_keys;
"
# Check for critical records
psql -h $DB_HOST -U spywatcher -d spywatcher -c "
SELECT * FROM \"User\" WHERE role = 'ADMIN' LIMIT 5;
"
```
```bash
# Run data integrity checks
psql -h $DB_HOST -U spywatcher -d spywatcher -c "
SELECT
(SELECT COUNT(*) FROM \"User\") as users,
(SELECT COUNT(*) FROM \"Guild\") as guilds,
(SELECT COUNT(*) FROM \"ApiKey\") as api_keys;
"
# Check for critical records
psql -h $DB_HOST -U spywatcher -d spywatcher -c "
SELECT * FROM \"User\" WHERE role = 'ADMIN' LIMIT 5;
"
```
6. **Restart Application** (15 minutes)
```bash
# If using Kubernetes
kubectl scale deployment spywatcher-backend --replicas=3
# If using Docker Compose
docker-compose up -d backend
# If using systemd
sudo systemctl start spywatcher-backend
```
```bash
# If using Kubernetes
kubectl scale deployment spywatcher-backend --replicas=3
# If using Docker Compose
docker-compose up -d backend
# If using systemd
sudo systemctl start spywatcher-backend
```
7. **Monitor for Errors** (20 minutes)
```bash
# Watch application logs
kubectl logs -f deployment/spywatcher-backend
# Or with Docker
docker-compose logs -f backend
# Check health endpoint
curl https://api.spywatcher.com/health
```
```bash
# Watch application logs
kubectl logs -f deployment/spywatcher-backend
# Or with Docker
docker-compose logs -f backend
# Check health endpoint
curl https://api.spywatcher.com/health
```
8. **Post-Recovery Verification** (10 minutes)
- Test critical API endpoints
- Verify user logins
- Check data consistency
- Monitor error rates in Sentry
- Verify Discord bot connectivity
- Test critical API endpoints
- Verify user logins
- Check data consistency
- Monitor error rates in Sentry
- Verify Discord bot connectivity
**Total RTO: ~2 hours**
### Scenario 2: Complete Infrastructure Failure
**Symptoms:**
- All services down
- Cannot access servers
- Cloud provider outage
@@ -182,97 +193,103 @@ Our backup strategy includes:
**Recovery Steps:**
1. **Assess Infrastructure Status** (15 minutes)
- Check cloud provider status page
- Verify network connectivity
- Identify affected resources
- Contact cloud support if needed
- Check cloud provider status page
- Verify network connectivity
- Identify affected resources
- Contact cloud support if needed
2. **Activate Disaster Recovery Site** (30 minutes)
```bash
# If using Terraform
cd infrastructure/
# Initialize Terraform with DR workspace
terraform workspace select disaster-recovery
# Review planned changes
terraform plan -out=dr.tfplan
# Apply infrastructure
terraform apply dr.tfplan
```
```bash
# If using Terraform
cd infrastructure/
# Initialize Terraform with DR workspace
terraform workspace select disaster-recovery
# Review planned changes
terraform plan -out=dr.tfplan
# Apply infrastructure
terraform apply dr.tfplan
```
3. **Restore Database in New Environment** (90 minutes)
```bash
# Set new environment variables
export DB_HOST="new-db-host.region.rds.amazonaws.com"
export S3_BUCKET="spywatcher-backups"
# Restore from secondary backup location
cd $PROJECT_ROOT/scripts
./restore.sh s3://spywatcher-backups-us-west/postgres/full/latest.dump.gz
```
```bash
# Set new environment variables
export DB_HOST="new-db-host.region.rds.amazonaws.com"
export S3_BUCKET="spywatcher-backups"
# Restore from secondary backup location
cd $PROJECT_ROOT/scripts
./restore.sh s3://spywatcher-backups-us-west/postgres/full/latest.dump.gz
```
4. **Deploy Application Containers** (45 minutes)
```bash
# If using Kubernetes
kubectl config use-context disaster-recovery
# Apply Kubernetes manifests
kubectl apply -f k8s/namespace.yaml
kubectl apply -f k8s/secrets.yaml
kubectl apply -f k8s/configmaps.yaml
kubectl apply -f k8s/deployments.yaml
kubectl apply -f k8s/services.yaml
kubectl apply -f k8s/ingress.yaml
# If using Docker Compose
docker-compose -f docker-compose.prod.yml up -d
```
```bash
# If using Kubernetes
kubectl config use-context disaster-recovery
# Apply Kubernetes manifests
kubectl apply -f k8s/namespace.yaml
kubectl apply -f k8s/secrets.yaml
kubectl apply -f k8s/configmaps.yaml
kubectl apply -f k8s/deployments.yaml
kubectl apply -f k8s/services.yaml
kubectl apply -f k8s/ingress.yaml
# If using Docker Compose
docker-compose -f docker-compose.prod.yml up -d
```
5. **Update DNS Records** (15 minutes)
```bash
# Update DNS to point to new infrastructure
# This depends on your DNS provider
# Example with AWS Route53:
aws route53 change-resource-record-sets \
--hosted-zone-id Z1234567890ABC \
--change-batch file://dns-update.json
```
```bash
# Update DNS to point to new infrastructure
# This depends on your DNS provider
# Example with AWS Route53:
aws route53 change-resource-record-sets \
--hosted-zone-id Z1234567890ABC \
--change-batch file://dns-update.json
```
6. **Run Smoke Tests** (20 minutes)
```bash
# Test critical endpoints
curl https://api.spywatcher.com/health
curl https://api.spywatcher.com/api/status
# Test authentication
curl -X POST https://api.spywatcher.com/api/auth/login \
-H "Content-Type: application/json" \
-d '{"username": "test", "password": "test"}'
# Test Discord bot
# (Check bot status in Discord server)
```
```bash
# Test critical endpoints
curl https://api.spywatcher.com/health
curl https://api.spywatcher.com/api/status
# Test authentication
curl -X POST https://api.spywatcher.com/api/auth/login \
-H "Content-Type: application/json" \
-d '{"username": "test", "password": "test"}'
# Test Discord bot
# (Check bot status in Discord server)
```
7. **Monitor System Health** (20 minutes)
- Check all services are running
- Verify database connections
- Monitor error rates
- Check Discord bot presence
- Verify frontend accessibility
- Check all services are running
- Verify database connections
- Monitor error rates
- Check Discord bot presence
- Verify frontend accessibility
8. **Notify Stakeholders**
- Update status page
- Send notification to users
- Post in Discord/Slack channels
- Document incident for post-mortem
- Update status page
- Send notification to users
- Post in Discord/Slack channels
- Document incident for post-mortem
**Total RTO: ~4 hours**
### Scenario 3: Regional Outage
**Symptoms:**
- Primary region unavailable
- High latency to primary services
- Cloud provider regional outage
@@ -280,66 +297,71 @@ Our backup strategy includes:
**Recovery Steps:**
1. **Confirm Regional Outage** (10 minutes)
- Check cloud provider status page
- Verify other regions are operational
- Assess blast radius
- Check cloud provider status page
- Verify other regions are operational
- Assess blast radius
2. **Activate Secondary Region** (30 minutes)
```bash
# Switch to secondary region infrastructure
cd infrastructure/
terraform workspace select us-west-2
terraform apply
```
```bash
# Switch to secondary region infrastructure
cd infrastructure/
terraform workspace select us-west-2
terraform apply
```
3. **Restore Database in Secondary Region** (90 minutes)
```bash
# Use secondary backup location
export DB_HOST="secondary-db.us-west-2.rds.amazonaws.com"
export S3_BUCKET="spywatcher-backups-us-west"
cd $PROJECT_ROOT/scripts
./restore.sh s3://spywatcher-backups-us-west/postgres/full/latest.dump.gz
```
```bash
# Use secondary backup location
export DB_HOST="secondary-db.us-west-2.rds.amazonaws.com"
export S3_BUCKET="spywatcher-backups-us-west"
cd $PROJECT_ROOT/scripts
./restore.sh s3://spywatcher-backups-us-west/postgres/full/latest.dump.gz
```
4. **Deploy to Secondary Region** (60 minutes)
```bash
# Deploy application to secondary region
kubectl config use-context us-west-2
kubectl apply -f k8s/
# Wait for pods to be ready
kubectl wait --for=condition=ready pod -l app=spywatcher-backend --timeout=300s
```
```bash
# Deploy application to secondary region
kubectl config use-context us-west-2
kubectl apply -f k8s/
# Wait for pods to be ready
kubectl wait --for=condition=ready pod -l app=spywatcher-backend --timeout=300s
```
5. **Update Global DNS** (30 minutes)
```bash
# Update DNS to point to secondary region
aws route53 change-resource-record-sets \
--hosted-zone-id Z1234567890ABC \
--change-batch file://failover-to-west.json
# Verify DNS propagation
dig api.spywatcher.com +short
```
```bash
# Update DNS to point to secondary region
aws route53 change-resource-record-sets \
--hosted-zone-id Z1234567890ABC \
--change-batch file://failover-to-west.json
# Verify DNS propagation
dig api.spywatcher.com +short
```
6. **Monitor Service Restoration** (20 minutes)
- Verify all services are healthy
- Check database replication lag (if applicable)
- Monitor error rates
- Verify user access
- Verify all services are healthy
- Check database replication lag (if applicable)
- Monitor error rates
- Verify user access
7. **Plan for Failback** (When primary region recovers)
- Schedule maintenance window
- Reverse failover procedure
- Update DNS back to primary region
- Run full system tests
- Schedule maintenance window
- Reverse failover procedure
- Update DNS back to primary region
- Run full system tests
**Total RTO: ~6 hours**
### Scenario 4: Ransomware Attack
**Symptoms:**
- Encrypted files
- Ransom notes
- Unusual file modifications
@@ -348,62 +370,64 @@ Our backup strategy includes:
**Recovery Steps:**
1. **Contain the Attack** (Immediate)
```bash
# Isolate affected systems
# Disable network access
# Revoke compromised credentials
# If using AWS
aws ec2 modify-instance-attribute \
--instance-id i-1234567890abcdef0 \
--no-source-dest-check
```
```bash
# Isolate affected systems
# Disable network access
# Revoke compromised credentials
# If using AWS
aws ec2 modify-instance-attribute \
--instance-id i-1234567890abcdef0 \
--no-source-dest-check
```
2. **Assess Impact** (30 minutes)
- Identify compromised systems
- Determine data loss
- Check backup integrity
- Review security logs
- Identify compromised systems
- Determine data loss
- Check backup integrity
- Review security logs
3. **Contact Security Team** (15 minutes)
- Notify security team
- Contact law enforcement if required
- Engage incident response team
- Preserve evidence
- Notify security team
- Contact law enforcement if required
- Engage incident response team
- Preserve evidence
4. **Restore from Clean Backup** (90 minutes)
```bash
# Use backup from before attack
# Verify backup is not compromised
cd $PROJECT_ROOT/scripts
# Identify clean backup (before attack)
aws s3 ls s3://spywatcher-backups/postgres/full/ | \
grep "2024-01-20" # Date before attack
# Restore clean backup
./restore.sh s3://spywatcher-backups/postgres/full/spywatcher_full_20240120_020000.dump.gz
```
```bash
# Use backup from before attack
# Verify backup is not compromised
cd $PROJECT_ROOT/scripts
# Identify clean backup (before attack)
aws s3 ls s3://spywatcher-backups/postgres/full/ | \
grep "2024-01-20" # Date before attack
# Restore clean backup
./restore.sh s3://spywatcher-backups/postgres/full/spywatcher_full_20240120_020000.dump.gz
```
5. **Rebuild Infrastructure** (120 minutes)
- Provision new clean infrastructure
- Apply security patches
- Update all credentials
- Implement additional security controls
- Provision new clean infrastructure
- Apply security patches
- Update all credentials
- Implement additional security controls
6. **Restore Service** (45 minutes)
- Deploy application to clean infrastructure
- Verify all security measures
- Enable monitoring and alerting
- Test thoroughly before full restoration
- Deploy application to clean infrastructure
- Verify all security measures
- Enable monitoring and alerting
- Test thoroughly before full restoration
7. **Post-Incident Actions**
- Conduct forensic analysis
- Update security policies
- Implement additional controls
- Train team on security awareness
- Schedule security audit
- Conduct forensic analysis
- Update security policies
- Implement additional controls
- Train team on security awareness
- Schedule security audit
**Total RTO: ~3 hours (excluding investigation time)**
@@ -418,6 +442,7 @@ cd $PROJECT_ROOT/scripts
```
**Requirements:**
- WAL archiving must be enabled
- WAL files must be available in S3
- Backup must be from before the target time
@@ -425,18 +450,21 @@ cd $PROJECT_ROOT/scripts
## Testing Schedule
### Monthly Tests
- [ ] Restore from latest backup to test database
- [ ] Verify backup integrity
- [ ] Test backup decryption
- [ ] Validate data completeness
### Quarterly Drills
- [ ] Full disaster recovery drill
- [ ] Document time to recovery
- [ ] Update procedures based on findings
- [ ] Train team members
### Annual Review
- [ ] Review and update RTO/RPO targets
- [ ] Update contact information
- [ ] Review and update procedures
@@ -445,18 +473,21 @@ cd $PROJECT_ROOT/scripts
## Contacts and Escalation
### Primary Contacts
- **Database Admin**: db-admin@spywatcher.com
- **DevOps Lead**: devops@spywatcher.com
- **Security Team**: security@spywatcher.com
- **On-Call Engineer**: oncall@spywatcher.com
### Escalation Path
1. On-Call Engineer (0-30 minutes)
2. Team Lead (30-60 minutes)
3. Engineering Manager (1-2 hours)
4. CTO (2+ hours)
### External Contacts
- **Cloud Provider Support**: support@aws.com
- **Database Vendor**: support@postgresql.org
- **Security Incident Response**: incident@security-firm.com
@@ -464,12 +495,14 @@ cd $PROJECT_ROOT/scripts
## Monitoring and Alerts
### Critical Alerts
- Backup failure alerts (via PagerDuty)
- Database health alerts
- Service availability alerts
- Security incident alerts
### Alert Channels
- **Email**: alerts@spywatcher.com
- **Slack**: #production-alerts
- **Discord**: #ops-alerts

120
DOCKER.md
View File

@@ -11,48 +11,55 @@ This guide explains how to run the Discord Spywatcher application using Docker a
## Quick Start (Development)
1. **Clone the repository**
```bash
git clone https://github.com/subculture-collective/discord-spywatcher.git
cd discord-spywatcher
```
```bash
git clone https://github.com/subculture-collective/discord-spywatcher.git
cd discord-spywatcher
```
2. **Create environment file**
```bash
cp .env.example .env
```
Edit `.env` and fill in your Discord credentials and other required values.
```bash
cp .env.example .env
```
Edit `.env` and fill in your Discord credentials and other required values.
3. **Start the development environment**
```bash
docker-compose -f docker-compose.dev.yml up
```
```bash
docker-compose -f docker-compose.dev.yml up
```
4. **Access the application**
- Frontend: http://localhost:5173
- Backend API: http://localhost:3001
- PostgreSQL: localhost:5432
- Frontend: http://localhost:5173
- Backend API: http://localhost:3001
- PostgreSQL: localhost:5432
## Environment Setup
### Development Environment
The development environment includes:
- **PostgreSQL 15**: Database with persistent volumes
- **Backend**: Node.js API with hot reload
- **Frontend**: Vite dev server with hot module replacement
**Start development environment:**
```bash
docker-compose -f docker-compose.dev.yml up
```
**Stop development environment:**
```bash
docker-compose -f docker-compose.dev.yml down
```
**Stop and remove volumes (clean start):**
```bash
docker-compose -f docker-compose.dev.yml down -v
```
@@ -60,22 +67,26 @@ docker-compose -f docker-compose.dev.yml down -v
### Production Environment
The production environment includes:
- **PostgreSQL 15**: Production database
- **Backend**: Optimized Node.js API
- **Frontend**: Nginx serving static files
- **Nginx**: Reverse proxy with SSL support
**Build and start production environment:**
```bash
docker-compose -f docker-compose.prod.yml up -d
```
**View logs:**
```bash
docker-compose -f docker-compose.prod.yml logs -f
```
**Stop production environment:**
```bash
docker-compose -f docker-compose.prod.yml down
```
@@ -85,6 +96,7 @@ docker-compose -f docker-compose.prod.yml down
The testing environment runs all tests in isolated containers:
**Run all tests:**
```bash
docker-compose -f docker-compose.test.yml up --abort-on-container-exit
```
@@ -94,16 +106,19 @@ docker-compose -f docker-compose.test.yml up --abort-on-container-exit
### Building Images
**Build all images:**
```bash
docker-compose -f docker-compose.dev.yml build
```
**Build specific service:**
```bash
docker-compose -f docker-compose.dev.yml build backend
```
**Build without cache:**
```bash
docker-compose -f docker-compose.dev.yml build --no-cache
```
@@ -111,26 +126,31 @@ docker-compose -f docker-compose.dev.yml build --no-cache
### Managing Containers
**Start services in background:**
```bash
docker-compose -f docker-compose.dev.yml up -d
```
**View running containers:**
```bash
docker-compose -f docker-compose.dev.yml ps
```
**View logs:**
```bash
docker-compose -f docker-compose.dev.yml logs -f [service_name]
```
**Restart a service:**
```bash
docker-compose -f docker-compose.dev.yml restart backend
```
**Execute commands in a container:**
```bash
docker-compose -f docker-compose.dev.yml exec backend sh
```
@@ -138,31 +158,37 @@ docker-compose -f docker-compose.dev.yml exec backend sh
### Database Management
**Run Prisma migrations:**
```bash
docker-compose -f docker-compose.dev.yml exec backend npx prisma migrate dev
```
**Generate Prisma Client:**
```bash
docker-compose -f docker-compose.dev.yml exec backend npx prisma generate
```
**Open Prisma Studio:**
```bash
docker-compose -f docker-compose.dev.yml exec backend npx prisma studio
```
**Access PostgreSQL CLI:**
```bash
docker-compose -f docker-compose.dev.yml exec postgres psql -U spywatcher -d spywatcher
```
**Backup database:**
```bash
docker-compose -f docker-compose.dev.yml exec postgres pg_dump -U spywatcher spywatcher > backup.sql
```
**Restore database:**
```bash
docker-compose -f docker-compose.dev.yml exec -T postgres psql -U spywatcher -d spywatcher < backup.sql
```
@@ -172,12 +198,14 @@ docker-compose -f docker-compose.dev.yml exec -T postgres psql -U spywatcher -d
### Hot Reload
Both frontend and backend support hot reload in development mode:
- **Backend**: Changes to `.ts` files automatically restart the server
- **Frontend**: Changes are reflected instantly via Vite HMR
### Installing New Dependencies
**Backend:**
```bash
# Stop containers
docker-compose -f docker-compose.dev.yml down
@@ -193,6 +221,7 @@ docker-compose -f docker-compose.dev.yml up
```
**Frontend:**
```bash
# Stop containers
docker-compose -f docker-compose.dev.yml down
@@ -210,17 +239,20 @@ docker-compose -f docker-compose.dev.yml up
### Running Backend Tests
**Run all backend tests:**
```bash
docker-compose -f docker-compose.dev.yml exec backend npm test
```
**Run specific test suite:**
```bash
docker-compose -f docker-compose.dev.yml exec backend npm run test:unit
docker-compose -f docker-compose.dev.yml exec backend npm run test:integration
```
**Run tests with coverage:**
```bash
docker-compose -f docker-compose.dev.yml exec backend npm run test:coverage
```
@@ -228,11 +260,13 @@ docker-compose -f docker-compose.dev.yml exec backend npm run test:coverage
### Running Frontend Tests
**Run all frontend tests:**
```bash
docker-compose -f docker-compose.dev.yml exec frontend npm test
```
**Run E2E tests:**
```bash
docker-compose -f docker-compose.dev.yml exec frontend npm run test:e2e
```
@@ -242,12 +276,14 @@ docker-compose -f docker-compose.dev.yml exec frontend npm run test:e2e
### Image Optimization
Production images are optimized using:
- Multi-stage builds
- Layer caching
- Minimal base images (Alpine Linux)
- Non-root user execution
**Check image sizes:**
```bash
docker images | grep spywatcher
```
@@ -255,11 +291,13 @@ docker images | grep spywatcher
### Health Checks
All services include health checks:
- **Backend**: `GET /api/health`
- **Frontend**: `GET /health`
- **PostgreSQL**: `pg_isready`
**Check service health:**
```bash
docker-compose -f docker-compose.prod.yml ps
```
@@ -267,6 +305,7 @@ docker-compose -f docker-compose.prod.yml ps
### Resource Limits
Production compose file includes resource limits:
- **Backend**: 1 CPU, 512MB RAM
- **Frontend**: 0.5 CPU, 256MB RAM
- **PostgreSQL**: 1 CPU, 512MB RAM
@@ -296,13 +335,14 @@ VITE_DISCORD_CLIENT_ID=your_client_id
For production with SSL:
1. Create SSL certificates directory:
```bash
mkdir -p nginx/ssl
```
```bash
mkdir -p nginx/ssl
```
2. Place your SSL certificates in `nginx/ssl/`:
- `nginx/ssl/cert.pem`
- `nginx/ssl/key.pem`
- `nginx/ssl/cert.pem`
- `nginx/ssl/key.pem`
3. Update `nginx/nginx.conf` for SSL configuration
@@ -314,32 +354,35 @@ If ports are already in use, modify the port mappings in `docker-compose.*.yml`:
```yaml
ports:
- "3002:3001" # Map host port 3002 to container port 3001
- '3002:3001' # Map host port 3002 to container port 3001
```
### Container Won't Start
1. Check logs:
```bash
docker-compose -f docker-compose.dev.yml logs [service_name]
```
```bash
docker-compose -f docker-compose.dev.yml logs [service_name]
```
2. Check container status:
```bash
docker-compose -f docker-compose.dev.yml ps
```
```bash
docker-compose -f docker-compose.dev.yml ps
```
3. Rebuild without cache:
```bash
docker-compose -f docker-compose.dev.yml build --no-cache
```
```bash
docker-compose -f docker-compose.dev.yml build --no-cache
```
### Database Connection Issues
1. Ensure PostgreSQL is healthy:
```bash
docker-compose -f docker-compose.dev.yml exec postgres pg_isready -U spywatcher
```
```bash
docker-compose -f docker-compose.dev.yml exec postgres pg_isready -U spywatcher
```
2. Check DATABASE_URL environment variable
3. Verify network connectivity between services
@@ -388,6 +431,7 @@ Add to `~/.bashrc` or `~/.zshrc` for persistence.
### Layer Caching
The Dockerfiles are optimized for layer caching:
1. Package files are copied first
2. Dependencies are installed
3. Source code is copied last
@@ -416,6 +460,7 @@ The repository includes a comprehensive Docker build workflow (`.github/workflow
5. **Reports image sizes** as PR comments
The workflow runs on:
- Pushes to main/develop branches
- Pull requests targeting main/develop
- Changes to Docker-related files
@@ -444,12 +489,14 @@ docker-compose -f docker-compose.test.yml up --abort-on-container-exit
### Image Registry
**Tag and push to Docker Hub:**
```bash
docker tag spywatcher-backend:latest your-username/spywatcher-backend:latest
docker push your-username/spywatcher-backend:latest
```
**Tag and push to GitHub Container Registry:**
```bash
docker tag spywatcher-backend:latest ghcr.io/your-org/spywatcher-backend:latest
docker push ghcr.io/your-org/spywatcher-backend:latest
@@ -461,9 +508,9 @@ docker push ghcr.io/your-org/spywatcher-backend:latest
2. **Use strong passwords** for database and secrets
3. **Keep base images updated** - Regularly rebuild with latest Alpine/Node images
4. **Scan for vulnerabilities**:
```bash
docker scan spywatcher-backend:latest
```
```bash
docker scan spywatcher-backend:latest
```
5. **Run as non-root user** - All production images use non-root users
6. **Use secret management** - For production, consider Docker secrets or external secret managers
@@ -477,6 +524,7 @@ docker push ghcr.io/your-org/spywatcher-backend:latest
## Support
For issues or questions:
- Open an issue on GitHub
- Check existing issues and discussions
- Review the main README.md for general setup

218
HEALTH_STATUS.md
View File

@@ -26,8 +26,8 @@ Checks if the service is running. Always returns 200 if the server is up.
```json
{
"status": "ok",
"timestamp": "2024-01-01T00:00:00.000Z"
"status": "ok",
"timestamp": "2024-01-01T00:00:00.000Z"
}
```
@@ -45,13 +45,13 @@ Checks if the service is ready to handle requests by verifying:
```json
{
"status": "healthy",
"checks": {
"database": true,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
"status": "healthy",
"checks": {
"database": true,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
}
```
@@ -59,13 +59,13 @@ Checks if the service is ready to handle requests by verifying:
```json
{
"status": "unhealthy",
"checks": {
"database": false,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
"status": "unhealthy",
"checks": {
"database": false,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
}
```
@@ -81,32 +81,32 @@ Returns current system status, uptime statistics, and active incidents.
```json
{
"status": "healthy",
"timestamp": "2024-01-01T00:00:00.000Z",
"services": {
"database": {
"status": "operational",
"latency": 10
"status": "healthy",
"timestamp": "2024-01-01T00:00:00.000Z",
"services": {
"database": {
"status": "operational",
"latency": 10
},
"redis": {
"status": "operational",
"latency": 5
},
"discord": {
"status": "operational",
"latency": 50
}
},
"redis": {
"status": "operational",
"latency": 5
"uptime": {
"24h": 99.9,
"7d": 99.5,
"30d": 99.0
},
"discord": {
"status": "operational",
"latency": 50
"incidents": {
"active": 0,
"critical": 0,
"major": 0
}
},
"uptime": {
"24h": 99.9,
"7d": 99.5,
"30d": 99.0
},
"incidents": {
"active": 0,
"critical": 0,
"major": 0
}
}
```
@@ -131,30 +131,30 @@ Returns historical status data for uptime charts.
```json
{
"period": {
"hours": 24,
"since": "2024-01-01T00:00:00.000Z"
},
"uptime": 99.9,
"checks": 288,
"avgLatency": {
"database": 12.5,
"redis": 6.2,
"discord": 52.1
},
"history": [
{
"timestamp": "2024-01-01T00:00:00.000Z",
"status": "healthy",
"overall": true,
"database": true,
"databaseLatency": 10,
"redis": true,
"redisLatency": 5,
"discord": true,
"discordLatency": 50
}
]
"period": {
"hours": 24,
"since": "2024-01-01T00:00:00.000Z"
},
"uptime": 99.9,
"checks": 288,
"avgLatency": {
"database": 12.5,
"redis": 6.2,
"discord": 52.1
},
"history": [
{
"timestamp": "2024-01-01T00:00:00.000Z",
"status": "healthy",
"overall": true,
"database": true,
"databaseLatency": 10,
"redis": true,
"redisLatency": 5,
"discord": true,
"discordLatency": 50
}
]
}
```
@@ -173,27 +173,27 @@ Returns list of incidents.
```json
{
"incidents": [
{
"id": "1",
"title": "Database Latency Issues",
"description": "Investigating high database latency",
"status": "INVESTIGATING",
"severity": "MAJOR",
"startedAt": "2024-01-01T00:00:00.000Z",
"resolvedAt": null,
"affectedServices": ["database"],
"updates": [
"incidents": [
{
"id": "u1",
"message": "We are investigating the issue",
"status": "INVESTIGATING",
"createdAt": "2024-01-01T00:05:00.000Z"
"id": "1",
"title": "Database Latency Issues",
"description": "Investigating high database latency",
"status": "INVESTIGATING",
"severity": "MAJOR",
"startedAt": "2024-01-01T00:00:00.000Z",
"resolvedAt": null,
"affectedServices": ["database"],
"updates": [
{
"id": "u1",
"message": "We are investigating the issue",
"status": "INVESTIGATING",
"createdAt": "2024-01-01T00:05:00.000Z"
}
]
}
]
}
],
"count": 1
],
"count": 1
}
```
@@ -228,12 +228,12 @@ All admin endpoints require authentication and admin role.
```json
{
"title": "Database Outage",
"description": "Database is experiencing connectivity issues",
"severity": "CRITICAL",
"status": "INVESTIGATING",
"affectedServices": ["database"],
"initialUpdate": "We are investigating the issue"
"title": "Database Outage",
"description": "Database is experiencing connectivity issues",
"severity": "CRITICAL",
"status": "INVESTIGATING",
"affectedServices": ["database"],
"initialUpdate": "We are investigating the issue"
}
```
@@ -257,12 +257,12 @@ All admin endpoints require authentication and admin role.
```json
{
"title": "Updated Title",
"description": "Updated description",
"severity": "MAJOR",
"status": "IDENTIFIED",
"affectedServices": ["database", "api"],
"updateMessage": "We have identified the root cause"
"title": "Updated Title",
"description": "Updated description",
"severity": "MAJOR",
"status": "IDENTIFIED",
"affectedServices": ["database", "api"],
"updateMessage": "We have identified the root cause"
}
```
@@ -279,8 +279,8 @@ All admin endpoints require authentication and admin role.
```json
{
"message": "Issue has been resolved",
"status": "RESOLVED"
"message": "Issue has been resolved",
"status": "RESOLVED"
}
```
@@ -403,19 +403,19 @@ No additional environment variables required. The feature uses existing database
```yaml
livenessProbe:
httpGet:
path: /health/live
port: 3001
initialDelaySeconds: 30
periodSeconds: 10
httpGet:
path: /health/live
port: 3001
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health/ready
port: 3001
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 3
httpGet:
path: /health/ready
port: 3001
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 3
```
### Docker Health Checks
@@ -483,14 +483,14 @@ The existing `/metrics` endpoint includes system health metrics. You can add ale
expr: (healthy_checks / total_checks) < 0.95
for: 5m
annotations:
summary: "Uptime below 95% in last 24 hours"
summary: 'Uptime below 95% in last 24 hours'
# Alert on high latency
- alert: HighDatabaseLatency
expr: avg_database_latency_ms > 100
for: 10m
annotations:
summary: "Database latency above 100ms"
summary: 'Database latency above 100ms'
```
## Best Practices

109
INFRASTRUCTURE.md
View File

@@ -61,28 +61,31 @@
## Components
### Compute
- **EKS Cluster**: Managed Kubernetes cluster (v1.28)
- **Node Groups**: Auto-scaling EC2 instances (t3.large)
- **Pods**: Containerized applications with health checks
### Networking
- **VPC**: Isolated network (10.0.0.0/16)
- **Subnets**: Public, Private, and Database across 3 AZs
- **NAT Gateways**: Internet access for private subnets
- **ALB**: HTTPS termination and routing
### Data Storage
- **RDS PostgreSQL**: Managed database (15.3)
- Multi-AZ for high availability
- Automated backups (7 days retention)
- Encryption at rest (KMS)
- Multi-AZ for high availability
- Automated backups (7 days retention)
- Encryption at rest (KMS)
- **ElastiCache Redis**: In-memory cache (7.0)
- Authentication token
- Encryption in transit
- Automatic failover
- Authentication token
- Encryption in transit
- Automatic failover
### Security
- **WAF**: Web Application Firewall with rate limiting
- **Security Groups**: Network-level access control
- **IAM Roles**: Fine-grained permissions
@@ -90,6 +93,7 @@
- **TLS/SSL**: End-to-end encryption
### Monitoring
- **CloudWatch**: Metrics, logs, and alarms
- **Health Checks**: Liveness and readiness probes
- **Resource Metrics**: CPU, memory, network usage
@@ -98,29 +102,30 @@
### Production Environment
| Component | Type | Specs | Replicas | Scaling |
|-----------|------|-------|----------|---------|
| Backend | Pod | 512Mi RAM, 500m CPU | 3 | 2-10 |
| Frontend | Pod | 128Mi RAM, 100m CPU | 2 | 2-5 |
| PostgreSQL | RDS | db.t3.large | 1 (Multi-AZ) | Manual |
| Redis | ElastiCache | cache.t3.medium | 2 | Manual |
| EKS Nodes | EC2 | t3.large | 3 | 2-10 |
| Component | Type | Specs | Replicas | Scaling |
| ---------- | ----------- | ------------------- | ------------ | ------- |
| Backend | Pod | 512Mi RAM, 500m CPU | 3 | 2-10 |
| Frontend | Pod | 128Mi RAM, 100m CPU | 2 | 2-5 |
| PostgreSQL | RDS | db.t3.large | 1 (Multi-AZ) | Manual |
| Redis | ElastiCache | cache.t3.medium | 2 | Manual |
| EKS Nodes | EC2 | t3.large | 3 | 2-10 |
### Staging Environment
| Component | Type | Specs | Replicas | Scaling |
|-----------|------|-------|----------|---------|
| Backend | Pod | 256Mi RAM, 250m CPU | 1 | 1-3 |
| Frontend | Pod | 128Mi RAM, 100m CPU | 1 | 1-2 |
| PostgreSQL | RDS | db.t3.medium | 1 | N/A |
| Redis | ElastiCache | cache.t3.small | 1 | N/A |
| EKS Nodes | EC2 | t3.medium | 2 | 1-4 |
| Component | Type | Specs | Replicas | Scaling |
| ---------- | ----------- | ------------------- | -------- | ------- |
| Backend | Pod | 256Mi RAM, 250m CPU | 1 | 1-3 |
| Frontend | Pod | 128Mi RAM, 100m CPU | 1 | 1-2 |
| PostgreSQL | RDS | db.t3.medium | 1 | N/A |
| Redis | ElastiCache | cache.t3.small | 1 | N/A |
| EKS Nodes | EC2 | t3.medium | 2 | 1-4 |
## Cost Estimation
### Monthly Costs (US East 1)
#### Production
- EKS Cluster: $73
- EC2 Nodes (3x t3.large): ~$150
- RDS PostgreSQL (db.t3.large, Multi-AZ): ~$290
@@ -132,6 +137,7 @@
**Total: ~$718/month**
#### Staging
- EKS Cluster: $73
- EC2 Nodes (2x t3.medium): ~$60
- RDS PostgreSQL (db.t3.medium): ~$70
@@ -141,23 +147,26 @@
**Total: ~$273/month**
*Note: Costs are estimates and may vary based on usage*
_Note: Costs are estimates and may vary based on usage_
## Deployment Strategies
### 1. Rolling Update (Default)
- **Use Case**: Standard deployments
- **Downtime**: Zero
- **Risk**: Low
- **Duration**: 5-10 minutes
### 2. Blue-Green
- **Use Case**: Major releases, critical changes
- **Downtime**: Zero
- **Risk**: Very Low (instant rollback)
- **Duration**: 10-15 minutes
### 3. Canary
- **Use Case**: High-risk changes, gradual rollout
- **Downtime**: Zero
- **Risk**: Minimal (gradual exposure)
@@ -166,18 +175,21 @@
## High Availability
### Application Layer
- Multiple replicas across availability zones
- Pod anti-affinity rules
- Pod disruption budgets (min 1 available)
- Health checks with automatic restart
### Database Layer
- Multi-AZ deployment for RDS
- Automated failover (< 60 seconds)
- Read replicas for scaling (optional)
- Point-in-time recovery
### Network Layer
- Multi-AZ load balancing
- Health checks on targets
- Automatic target deregistration
@@ -186,15 +198,18 @@
## Disaster Recovery
### RTO (Recovery Time Objective)
- Application: < 5 minutes
- Database: < 1 minute (automated failover)
- Full Infrastructure: < 30 minutes (Terraform redeploy)
### RPO (Recovery Point Objective)
- Database: < 5 minutes (automated backups)
- Application: 0 (stateless, recreatable)
### Backup Strategy
- **Database**: Daily automated backups (7 days retention)
- **Configuration**: Git repository (versioned)
- **Infrastructure**: Terraform state (versioned in S3)
@@ -202,24 +217,28 @@
## Security Measures
### Network Security
- Private subnets for application and database
- Security groups with least-privilege rules
- Network ACLs
- VPC Flow Logs
### Application Security
- Containers run as non-root
- Read-only root filesystems where possible
- No privilege escalation
- Security scanning in CI/CD
### Data Security
- Encryption at rest (KMS)
- Encryption in transit (TLS 1.2+)
- Secrets stored in AWS Secrets Manager
- Database credentials auto-rotated
### Access Control
- IAM roles with least privilege
- RBAC in Kubernetes
- MFA for admin access
@@ -228,17 +247,18 @@
## Scaling Strategy
### Horizontal Scaling
- **Triggers**:
- CPU > 70%
- Memory > 80%
- Custom metrics (request rate)
- CPU > 70%
- Memory > 80%
- Custom metrics (request rate)
- **Limits**:
- Backend: 2-10 pods
- Frontend: 2-5 pods
- Nodes: 2-10 instances
- Backend: 2-10 pods
- Frontend: 2-5 pods
- Nodes: 2-10 instances
### Vertical Scaling
- Database: Manual scaling with downtime
- Redis: Manual scaling with failover
- Pods: Update resource limits and restart
@@ -246,46 +266,51 @@
## Monitoring Strategy
### Application Metrics
- Request rate and latency
- Error rate
- Active connections
- Cache hit rate
### Infrastructure Metrics
- CPU utilization
- Memory utilization
- Network throughput
- Disk I/O
### Business Metrics
- Active users
- API usage per tier
- Feature usage
- User sessions
### Alerting
- Critical: Page immediately
- Service down
- Database unavailable
- High error rate
- Service down
- Database unavailable
- High error rate
- Warning: Notify during business hours
- High CPU/memory
- Low disk space
- Elevated response time
- High CPU/memory
- Low disk space
- Elevated response time
## Maintenance Windows
### Planned Maintenance
- **Schedule**: Sundays 02:00-04:00 UTC
- **Notification**: 7 days advance notice
- **Activities**:
- OS patches
- Database maintenance
- Kubernetes upgrades
- SSL certificate renewal
- OS patches
- Database maintenance
- Kubernetes upgrades
- SSL certificate renewal
### Emergency Maintenance
- Immediate security patches
- Critical bug fixes
- Infrastructure failures
@@ -293,17 +318,21 @@
## Compliance & Governance
### Tagging Strategy
All resources tagged with:
- `Environment`: production/staging
- `Project`: spywatcher
- `ManagedBy`: terraform
- `CostCenter`: engineering
### Resource Naming
- Pattern: `{project}-{environment}-{resource}`
- Example: `spywatcher-production-backend`
### Access Audit
- CloudTrail enabled
- Quarterly access review
- Regular security audits

165
LOGGING.md
View File

@@ -7,6 +7,7 @@ This document describes the centralized logging infrastructure for Discord SpyWa
Discord SpyWatcher implements a comprehensive log aggregation system that collects, stores, and analyzes logs from all services in a centralized location.
**Stack Components:**
- **Grafana Loki** - Log aggregation and storage system
- **Promtail** - Log collection and shipping agent
- **Grafana** - Visualization and search UI
@@ -46,6 +47,7 @@ Discord SpyWatcher implements a comprehensive log aggregation system that collec
## Features
### ✅ Log Collection
- **Backend logs** - Application, security, and error logs in JSON format
- **Security logs** - Authentication, authorization, and security events
- **Database logs** - PostgreSQL query and connection logs
@@ -55,17 +57,20 @@ Discord SpyWatcher implements a comprehensive log aggregation system that collec
- **Container logs** - Docker container stdout/stderr
### ✅ Structured Logging
- JSON format for easy parsing and filtering
- Request ID correlation for tracing
- Log levels: error, warn, info, debug
- Automatic metadata enrichment (service, job, level)
### ✅ Retention Policies
- **30-day retention** - Automatic deletion of logs older than 30 days
- **Compression** - Automatic log compression to save storage
- **Configurable** - Easy to adjust retention period based on requirements
### ✅ Search & Filtering
- **LogQL** - Powerful query language for log searching
- **Grafana UI** - User-friendly interface for log exploration
- **Filters** - Filter by service, level, time range, and custom fields
@@ -76,11 +81,13 @@ Discord SpyWatcher implements a comprehensive log aggregation system that collec
### Starting the Logging Stack
**Development:**
```bash
docker-compose -f docker-compose.dev.yml up -d loki promtail grafana
```
**Production:**
```bash
docker-compose -f docker-compose.prod.yml up -d loki promtail grafana
```
@@ -89,13 +96,14 @@ docker-compose -f docker-compose.prod.yml up -d loki promtail grafana
1. Open your browser to `http://localhost:3000`
2. Login with default credentials:
- Username: `admin`
- Password: `admin` (change on first login)
- Username: `admin`
- Password: `admin` (change on first login)
3. Navigate to **Explore** or **Dashboards** > **Spywatcher - Log Aggregation**
### Changing Admin Credentials
Set environment variables:
```bash
GRAFANA_ADMIN_USER=your_username
GRAFANA_ADMIN_PASSWORD=your_secure_password
@@ -108,6 +116,7 @@ GRAFANA_ADMIN_PASSWORD=your_secure_password
Location: `loki/loki-config.yml`
**Key settings:**
- `retention_period: 720h` - Keep logs for 30 days
- `ingestion_rate_mb: 15` - Max ingestion rate (15 MB/s)
- `max_entries_limit_per_query: 5000` - Max entries per query
@@ -117,12 +126,14 @@ Location: `loki/loki-config.yml`
Location: `promtail/promtail-config.yml`
**Log sources configured:**
- Backend application logs (`/logs/backend/*.log`)
- Security logs (`/logs/backend/security.log`)
- PostgreSQL logs (`/var/log/postgresql/*.log`)
- Docker container logs (via Docker socket)
**Pipeline stages:**
- JSON parsing for structured logs
- Label extraction (level, service, action, etc.)
- Timestamp parsing
@@ -133,10 +144,12 @@ Location: `promtail/promtail-config.yml`
Location: `grafana/provisioning/`
**Datasources:**
- Loki (default) - `http://loki:3100`
- Prometheus - `http://backend:3001/metrics`
**Dashboards:**
- `Spywatcher - Log Aggregation` - Main logging dashboard
## Usage
@@ -144,51 +157,61 @@ Location: `grafana/provisioning/`
### Searching Logs
#### Basic Search
```logql
{job="backend"}
```
#### Filter by Level
```logql
{job="backend", level="error"}
```
#### Search in Message
```logql
{job="backend"} |= "error"
```
#### Security Logs
```logql
{job="security"} | json | action="LOGIN_ATTEMPT"
```
#### Time Range
Use Grafana's time picker to select a specific time range (e.g., last 1 hour, last 24 hours, custom range).
### Common Queries
**All errors in the last hour:**
```logql
{job=~"backend|security"} | json | level="error"
```
**Failed login attempts:**
```logql
{job="security"} | json | action="LOGIN_ATTEMPT" | result="FAILURE"
```
**Slow database queries:**
```logql
{job="backend"} | json | message=~".*query.*" | duration > 1000
```
**Rate limiting events:**
```logql
{job="security"} | json | action="RATE_LIMIT_VIOLATION"
```
**Request by request ID:**
```logql
{job="backend"} | json | requestId="abc123"
```
@@ -213,6 +236,7 @@ The pre-configured dashboard includes:
5. **Error Logs** - Quick view of all error logs
**Template Variables:**
- `$job` - Filter by job (backend, security, postgres, etc.)
- `$level` - Filter by log level (error, warn, info, debug)
- `$search` - Free-text search filter
@@ -233,14 +257,14 @@ logger.info('User logged in', { userId: user.id });
import { logWithRequestId } from './middleware/winstonLogger';
logWithRequestId('info', 'Processing request', req.id, {
userId: user.id,
action: 'fetch_data'
userId: user.id,
action: 'fetch_data',
});
// Error logging
logger.error('Database connection failed', {
error: err.message,
stack: err.stack
error: err.message,
stack: err.stack,
});
```
@@ -259,12 +283,12 @@ Use the security logger for security-related events:
import { logSecurityEvent, SecurityActions } from './utils/securityLogger';
await logSecurityEvent({
userId: user.discordId,
action: SecurityActions.LOGIN_SUCCESS,
result: 'SUCCESS',
ipAddress: req.ip,
userAgent: req.get('user-agent'),
requestId: req.id
userId: user.discordId,
action: SecurityActions.LOGIN_SUCCESS,
result: 'SUCCESS',
ipAddress: req.ip,
userAgent: req.get('user-agent'),
requestId: req.id,
});
```
@@ -283,16 +307,17 @@ Edit `loki/loki-config.yml`:
```yaml
limits_config:
retention_period: 720h # Change this value (e.g., 1440h for 60 days)
retention_period: 720h # Change this value (e.g., 1440h for 60 days)
table_manager:
retention_period: 720h # Keep same as above
retention_period: 720h # Keep same as above
compactor:
retention_enabled: true
retention_enabled: true
```
Then restart Loki:
```bash
docker-compose restart loki
```
@@ -305,29 +330,29 @@ Adjust in `loki/loki-config.yml`:
```yaml
limits_config:
ingestion_rate_mb: 15 # MB/s per tenant
ingestion_burst_size_mb: 20 # Burst size
per_stream_rate_limit: 3MB # Per stream rate
per_stream_rate_limit_burst: 15MB # Per stream burst
ingestion_rate_mb: 15 # MB/s per tenant
ingestion_burst_size_mb: 20 # Burst size
per_stream_rate_limit: 3MB # Per stream rate
per_stream_rate_limit_burst: 15MB # Per stream burst
```
### Query Performance
```yaml
limits_config:
max_entries_limit_per_query: 5000 # Max entries returned
max_streams_per_user: 10000 # Max streams per user
max_entries_limit_per_query: 5000 # Max entries returned
max_streams_per_user: 10000 # Max streams per user
```
### Cache Configuration
```yaml
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100 # Increase for better performance
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100 # Increase for better performance
```
## Alerting
@@ -338,25 +363,25 @@ query_range:
```yaml
groups:
- name: spywatcher-alerts
interval: 1m
rules:
- alert: HighErrorRate
expr: |
sum(rate({job="backend", level="error"}[5m])) > 10
for: 5m
labels:
severity: critical
annotations:
summary: "High error rate detected"
description: "Error rate is {{ $value }} errors/sec"
- name: spywatcher-alerts
interval: 1m
rules:
- alert: HighErrorRate
expr: |
sum(rate({job="backend", level="error"}[5m])) > 10
for: 5m
labels:
severity: critical
annotations:
summary: 'High error rate detected'
description: 'Error rate is {{ $value }} errors/sec'
```
2. Configure Alertmanager URL in `loki/loki-config.yml`:
```yaml
ruler:
alertmanager_url: http://alertmanager:9093
alertmanager_url: http://alertmanager:9093
```
## Troubleshooting
@@ -364,34 +389,39 @@ ruler:
### Logs not appearing in Grafana
1. **Check Promtail is running:**
```bash
docker ps | grep promtail
docker logs spywatcher-promtail-dev
```
```bash
docker ps | grep promtail
docker logs spywatcher-promtail-dev
```
2. **Check Loki is accepting logs:**
```bash
curl http://localhost:3100/ready
```
```bash
curl http://localhost:3100/ready
```
3. **Verify log files exist:**
```bash
docker exec spywatcher-backend-dev ls -la /app/logs
```
```bash
docker exec spywatcher-backend-dev ls -la /app/logs
```
4. **Check Promtail configuration:**
```bash
docker exec spywatcher-promtail-dev cat /etc/promtail/config.yml
```
```bash
docker exec spywatcher-promtail-dev cat /etc/promtail/config.yml
```
### Loki storage issues
**Check disk usage:**
```bash
du -sh /var/lib/docker/volumes/discord-spywatcher_loki-data/
```
**Force compaction:**
```bash
docker exec spywatcher-loki-dev wget -qO- http://localhost:3100/loki/api/v1/delete?query={job="backend"}&start=2024-01-01T00:00:00Z&end=2024-01-02T00:00:00Z
```
@@ -410,6 +440,7 @@ docker exec spywatcher-loki-dev wget -qO- http://localhost:3100/loki/api/v1/dele
Available at: `http://localhost:3100/metrics`
**Key metrics:**
- `loki_ingester_chunks_created_total` - Chunks created
- `loki_ingester_bytes_received_total` - Bytes ingested
- `loki_request_duration_seconds` - Query performance
@@ -419,6 +450,7 @@ Available at: `http://localhost:3100/metrics`
Available at: `http://localhost:9080/metrics`
**Key metrics:**
- `promtail_sent_entries_total` - Entries sent to Loki
- `promtail_dropped_entries_total` - Dropped entries
- `promtail_read_bytes_total` - Bytes read from logs
@@ -443,12 +475,13 @@ Logs can reference Sentry issues:
```typescript
logger.error('Unhandled exception', {
sentryEventId: sentryEventId,
error: err.message
sentryEventId: sentryEventId,
error: err.message,
});
```
Search in Loki:
```logql
{job="backend"} | json | sentryEventId="abc123"
```
@@ -468,6 +501,7 @@ Search in Loki:
### Log Sanitization
Winston logger automatically sanitizes sensitive data:
- Passwords
- Tokens (access, refresh, API keys)
- OAuth scopes
@@ -484,31 +518,34 @@ See: `backend/src/utils/securityLogger.ts`
## Resources
### Documentation
- [Grafana Loki Documentation](https://grafana.com/docs/loki/latest/)
- [Promtail Documentation](https://grafana.com/docs/loki/latest/clients/promtail/)
- [LogQL Query Language](https://grafana.com/docs/loki/latest/logql/)
- [Grafana Documentation](https://grafana.com/docs/grafana/latest/)
### Example Queries
- [LogQL Examples](https://grafana.com/docs/loki/latest/logql/example-queries/)
- [Query Patterns](https://grafana.com/blog/2020/04/08/loki-log-queries/)
### Community
- [Loki GitHub Repository](https://github.com/grafana/loki)
- [Grafana Community Forums](https://community.grafana.com/)
## Comparison with ELK Stack
| Feature | Loki Stack | ELK Stack |
|---------|-----------|-----------|
| **Storage** | Index labels, not full text | Full text indexing |
| **Resource Usage** | Low (300-500MB) | High (2-4GB+) |
| **Query Language** | LogQL (Prometheus-like) | Lucene/KQL |
| **Setup Complexity** | Simple (3 containers) | Complex (5+ containers) |
| **Cost** | Free, open source | Free, but resource intensive |
| **Scalability** | Good for small-medium | Better for enterprise |
| **Integration** | Native Prometheus/Grafana | Elasticsearch ecosystem |
| **Best For** | Cloud-native, Kubernetes | Large enterprises, full-text search |
| Feature | Loki Stack | ELK Stack |
| -------------------- | --------------------------- | ----------------------------------- |
| **Storage** | Index labels, not full text | Full text indexing |
| **Resource Usage** | Low (300-500MB) | High (2-4GB+) |
| **Query Language** | LogQL (Prometheus-like) | Lucene/KQL |
| **Setup Complexity** | Simple (3 containers) | Complex (5+ containers) |
| **Cost** | Free, open source | Free, but resource intensive |
| **Scalability** | Good for small-medium | Better for enterprise |
| **Integration** | Native Prometheus/Grafana | Elasticsearch ecosystem |
| **Best For** | Cloud-native, Kubernetes | Large enterprises, full-text search |
## Conclusion

166
MIGRATION.md
View File

@@ -7,12 +7,14 @@ This guide explains how to migrate your Discord Spywatcher database from SQLite
The application now uses PostgreSQL as the primary database for production deployments. This provides:
### Standard Benefits
- Better concurrency handling
- Improved data integrity
- Enhanced scalability
- Production-ready features
### PostgreSQL-Specific Enhancements
- **JSONB Fields**: Flexible metadata storage with efficient querying
- **Array Types**: Native array support for multi-value fields (clients, roles)
- **UUID Primary Keys**: Better distribution and security for event models
@@ -25,15 +27,15 @@ The application now uses PostgreSQL as the primary database for production deplo
### What Changed in the Schema
1. **Event Models** (PresenceEvent, TypingEvent, MessageEvent, etc.):
- IDs changed from `Int` to `String` (UUID)
- Added `metadata Json? @db.JsonB` field
- Timestamps now use `@db.Timestamptz`
- Comma-separated strings converted to arrays
- IDs changed from `Int` to `String` (UUID)
- Added `metadata Json? @db.JsonB` field
- Timestamps now use `@db.Timestamptz`
- Comma-separated strings converted to arrays
2. **All Models**:
- All timestamps upgraded to timezone-aware
- Added strategic indexes for performance
- JSONB for all JSON fields
- All timestamps upgraded to timezone-aware
- Added strategic indexes for performance
- JSONB for all JSON fields
For complete PostgreSQL feature documentation, see [POSTGRESQL.md](./POSTGRESQL.md).
@@ -42,9 +44,10 @@ For complete PostgreSQL feature documentation, see [POSTGRESQL.md](./POSTGRESQL.
If you're starting fresh with Docker, no migration is needed. Simply:
1. Start the Docker environment:
```bash
docker-compose -f docker-compose.dev.yml up
```
```bash
docker-compose -f docker-compose.dev.yml up
```
2. The PostgreSQL database will be initialized automatically with all migrations.
@@ -98,6 +101,7 @@ SQLITE_DATABASE_URL="file:./prisma/dev.db" \
```
The migration script will:
- Convert integer IDs to UUIDs
- Transform comma-separated strings to arrays
- Batch process large datasets (1000 records at a time)
@@ -128,14 +132,16 @@ If your existing data is test data or not critical:
If your existing data is test data or not critical:
1. Backup your existing data (optional):
```bash
cp backend/prisma/dev.db backend/prisma/dev.db.backup
```
```bash
cp backend/prisma/dev.db backend/prisma/dev.db.backup
```
2. Start with Docker:
```bash
docker-compose -f docker-compose.dev.yml up
```
```bash
docker-compose -f docker-compose.dev.yml up
```
3. Your data will be in the new PostgreSQL database (empty initially).
@@ -158,27 +164,29 @@ npx prisma studio
#### Step 2: Transform and Import to PostgreSQL
1. Start PostgreSQL with Docker:
```bash
docker-compose -f docker-compose.dev.yml up postgres -d
```
```bash
docker-compose -f docker-compose.dev.yml up postgres -d
```
2. Run migrations on PostgreSQL:
```bash
docker-compose -f docker-compose.dev.yml exec postgres psql -U spywatcher -d spywatcher
```
```bash
docker-compose -f docker-compose.dev.yml exec postgres psql -U spywatcher -d spywatcher
```
3. Transform SQLite SQL to PostgreSQL format:
SQLite and PostgreSQL have syntax differences. You'll need to:
- Remove SQLite-specific syntax
- Adjust data types
- Handle AUTOINCREMENT → SERIAL/BIGSERIAL conversions
- Fix boolean values (0/1 → false/true)
SQLite and PostgreSQL have syntax differences. You'll need to:
- Remove SQLite-specific syntax
- Adjust data types
- Handle AUTOINCREMENT → SERIAL/BIGSERIAL conversions
- Fix boolean values (0/1 → false/true)
4. Import the transformed data:
```bash
docker-compose -f docker-compose.dev.yml exec -T postgres psql -U spywatcher -d spywatcher < postgres_import.sql
```
```bash
docker-compose -f docker-compose.dev.yml exec -T postgres psql -U spywatcher -d spywatcher < postgres_import.sql
```
#### Step 3: Verify Data
@@ -226,44 +234,54 @@ pgloader migrate.load
The schema has been significantly enhanced for PostgreSQL:
### Event Models (Breaking Changes)
All event models have been updated with PostgreSQL-specific features:
#### ID Fields
- **Before**: `id Int @id @default(autoincrement())`
- **After**: `id String @id @default(uuid())`
- **Impact**: IDs are now UUIDs instead of sequential integers
#### Array Fields
- **PresenceEvent.clients**:
- Before: `String` (comma-separated: "desktop,web")
- After: `String[]` (array: ["desktop", "web"])
- **PresenceEvent.clients**:
- Before: `String` (comma-separated: "desktop,web")
- After: `String[]` (array: ["desktop", "web"])
- **RoleChangeEvent.addedRoles**:
- Before: `String` (comma-separated role IDs)
- After: `String[]` (array of role IDs)
- Before: `String` (comma-separated role IDs)
- After: `String[]` (array of role IDs)
#### Metadata Fields
All event models now include:
```prisma
metadata Json? @db.JsonB
```
#### Timestamp Fields
- **Before**: `createdAt DateTime @default(now())`
- **After**: `createdAt DateTime @default(now()) @db.Timestamptz`
- **Impact**: Timezone-aware timestamps
### Guild Model
- **SQLite**: `permissions Int`
- **PostgreSQL**: `permissions BigInt`
- **Reason**: Discord permission values can exceed 32-bit integer limits
### User and Security Models
- All timestamps upgraded to `@db.Timestamptz`
- All JSON fields upgraded to `@db.JsonB`
- Additional indexes added for performance
### Full-Text Search
MessageEvent now supports full-text search via:
- Generated `content_search` tsvector column
- GIN index for efficient searches
- Setup via `npm run db:fulltext`
@@ -279,12 +297,12 @@ If your code directly accesses ID fields as integers, update to handle UUIDs:
```typescript
// Before
const event = await db.presenceEvent.findUnique({
where: { id: 123 }
where: { id: 123 },
});
// After
const event = await db.presenceEvent.findUnique({
where: { id: "550e8400-e29b-41d4-a716-446655440000" }
where: { id: '550e8400-e29b-41d4-a716-446655440000' },
});
```
@@ -303,25 +321,25 @@ const clients = event.clients; // Already an array
```typescript
// Store flexible data in metadata
await db.presenceEvent.create({
data: {
userId: "123",
username: "user",
clients: ["desktop", "mobile"],
metadata: {
status: "online",
customField: "value"
}
}
data: {
userId: '123',
username: 'user',
clients: ['desktop', 'mobile'],
metadata: {
status: 'online',
customField: 'value',
},
},
});
// Query JSONB data
const events = await db.presenceEvent.findMany({
where: {
metadata: {
path: ['status'],
equals: 'online'
}
}
where: {
metadata: {
path: ['status'],
equals: 'online',
},
},
});
```
@@ -384,6 +402,7 @@ For production deployment with PostgreSQL:
### 1. Set up PostgreSQL Database
Use a managed PostgreSQL service:
- AWS RDS
- Google Cloud SQL
- Azure Database for PostgreSQL
@@ -451,11 +470,13 @@ docker-compose -f docker-compose.prod.yml exec backend npx prisma studio
### Connection Issues
**Problem**: Cannot connect to PostgreSQL
```
Error: P1001: Can't reach database server at `postgres:5432`
```
**Solution**:
**Solution**:
- Ensure PostgreSQL container is running: `docker-compose -f docker-compose.dev.yml ps`
- Check network connectivity between containers
- Verify DATABASE_URL is correct
@@ -463,11 +484,13 @@ Error: P1001: Can't reach database server at `postgres:5432`
### Migration Failures
**Problem**: Migration fails with schema mismatch
```
Error: P3009: migrate found failed migrations
```
**Solution**:
```bash
# Reset migrations (WARNING: This will delete all data)
docker-compose -f docker-compose.dev.yml exec backend npx prisma migrate reset
@@ -479,30 +502,34 @@ docker-compose -f docker-compose.dev.yml exec backend npx prisma migrate resolve
### Permission Errors
**Problem**: Permission denied for PostgreSQL
```
Error: FATAL: password authentication failed for user "spywatcher"
```
**Solution**:
- Check DB_PASSWORD in `.env` matches PostgreSQL configuration
- Recreate PostgreSQL container with correct credentials:
```bash
docker-compose -f docker-compose.dev.yml down -v
docker-compose -f docker-compose.dev.yml up postgres -d
```
```bash
docker-compose -f docker-compose.dev.yml down -v
docker-compose -f docker-compose.dev.yml up postgres -d
```
### Data Type Issues
**Problem**: UUID type errors
```
Type 'number' is not assignable to type 'string'
```
**Solution**:
Update code to handle UUID strings instead of integers:
```typescript
// Use UUID strings
const id = "550e8400-e29b-41d4-a716-446655440000";
const id = '550e8400-e29b-41d4-a716-446655440000';
// Generate new UUIDs
import { randomUUID } from 'crypto';
@@ -510,12 +537,14 @@ const newId = randomUUID();
```
**Problem**: Array field errors
```
Cannot read property 'split' of undefined
```
**Solution**:
Update code to handle native arrays:
```typescript
// Before
const clients = event.clients.split(',');
@@ -525,15 +554,17 @@ const clients = event.clients; // Already an array
```
**Problem**: BigInt serialization errors in JavaScript
```
Do not know how to serialize a BigInt
```
**Solution**:
Add BigInt serialization support in your code:
```javascript
BigInt.prototype.toJSON = function() {
return this.toString();
BigInt.prototype.toJSON = function () {
return this.toString();
};
```
@@ -553,15 +584,18 @@ BigInt.prototype.toJSON = function() {
## Available Scripts and Tools
### Migration and Setup
- `npm run db:migrate` - Migrate data from SQLite to PostgreSQL
- `npm run db:migrate:dry` - Test migration without writing data
- `npm run db:fulltext` - Setup full-text search on messages
### Backup and Recovery
- `npm run db:backup` - Create compressed database backup
- `npm run db:restore <file>` - Restore from backup file
### Maintenance
- `npm run db:maintenance` - Run routine maintenance tasks
- `npx prisma studio` - Open visual database browser
- `npx prisma migrate deploy` - Apply pending migrations
@@ -574,20 +608,22 @@ BigInt.prototype.toJSON = function() {
- **PostgreSQL Documentation**: https://www.postgresql.org/docs/
- **Prisma PostgreSQL Guide**: https://www.prisma.io/docs/concepts/database-connectors/postgresql
- **PostgreSQL Performance Tuning**: https://wiki.postgresql.org/wiki/Performance_Optimization
5. **Update application code**: Ensure your application properly handles BigInt types for Discord permissions
## Support
For issues or questions about database migration:
- Check this guide and [POSTGRESQL.md](./POSTGRESQL.md)
- Review script documentation in [scripts/README.md](./scripts/README.md)
- Check [DOCKER.md](./DOCKER.md) for Docker-specific troubleshooting
- Review [Prisma Migration Docs](https://www.prisma.io/docs/concepts/components/prisma-migrate)
- Open an issue on GitHub with:
- Migration script output
- Error messages
- Database versions (SQLite and PostgreSQL)
- Row counts before and after migration
- Migration script output
- Error messages
- Database versions (SQLite and PostgreSQL)
- Row counts before and after migration
## Migration Checklist

130
MONITORING.md
View File

@@ -5,6 +5,7 @@ This document describes the monitoring and observability features implemented in
## Overview
The application includes comprehensive monitoring with:
- **Sentry** for error tracking and Application Performance Monitoring (APM)
- **Prometheus** for metrics collection
- **Winston** for structured logging
@@ -43,27 +44,31 @@ Prometheus metrics are exposed at the `/metrics` endpoint for scraping.
#### Available Metrics
**Default Metrics** (automatically collected):
- `process_cpu_*` - CPU usage metrics
- `process_resident_memory_bytes` - Memory usage
- `nodejs_*` - Node.js-specific metrics
- `nodejs_gc_*` - Garbage collection metrics
**Custom HTTP Metrics**:
- `http_request_duration_seconds` - HTTP request duration histogram
- Labels: `method`, `route`, `status_code`
- Buckets: 0.1s, 0.5s, 1s, 2s, 5s
- Labels: `method`, `route`, `status_code`
- Buckets: 0.1s, 0.5s, 1s, 2s, 5s
- `http_requests_total` - Total HTTP requests counter
- Labels: `method`, `route`, `status_code`
- Labels: `method`, `route`, `status_code`
- `http_requests_errors` - Total HTTP errors counter
- Labels: `method`, `route`, `status_code`
- Labels: `method`, `route`, `status_code`
**WebSocket Metrics**:
- `websocket_active_connections` - Current number of active WebSocket connections
**Database Metrics**:
- `db_query_duration_seconds` - Database query duration histogram
- Labels: `model`, `operation`
- Buckets: 0.01s, 0.05s, 0.1s, 0.5s, 1s
- Labels: `model`, `operation`
- Buckets: 0.01s, 0.05s, 0.1s, 0.5s, 1s
#### Accessing Metrics
@@ -75,11 +80,11 @@ curl http://localhost:3001/metrics
```yaml
scrape_configs:
- job_name: 'spywatcher'
static_configs:
- targets: ['localhost:3001']
metrics_path: '/metrics'
scrape_interval: 15s
- job_name: 'spywatcher'
static_configs:
- targets: ['localhost:3001']
metrics_path: '/metrics'
scrape_interval: 15s
```
### 3. Health Check Endpoints
@@ -93,10 +98,11 @@ Endpoint: `GET /health/live`
Checks if the service is running.
**Response (200 OK)**:
```json
{
"status": "ok",
"timestamp": "2024-01-01T00:00:00.000Z"
"status": "ok",
"timestamp": "2024-01-01T00:00:00.000Z"
}
```
@@ -105,33 +111,36 @@ Checks if the service is running.
Endpoint: `GET /health/ready`
Checks if the service is ready to handle requests by verifying:
- Database connectivity
- Redis connectivity (optional)
- Discord API connectivity
**Response (200 OK - all healthy)**:
```json
{
"status": "healthy",
"checks": {
"database": true,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
"status": "healthy",
"checks": {
"database": true,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
}
```
**Response (503 Service Unavailable - unhealthy)**:
```json
{
"status": "unhealthy",
"checks": {
"database": false,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
"status": "unhealthy",
"checks": {
"database": false,
"redis": true,
"discord": true
},
"timestamp": "2024-01-01T00:00:00.000Z"
}
```
@@ -139,18 +148,18 @@ Checks if the service is ready to handle requests by verifying:
```yaml
livenessProbe:
httpGet:
path: /health/live
port: 3001
initialDelaySeconds: 30
periodSeconds: 10
httpGet:
path: /health/live
port: 3001
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health/ready
port: 3001
initialDelaySeconds: 10
periodSeconds: 5
httpGet:
path: /health/ready
port: 3001
initialDelaySeconds: 10
periodSeconds: 5
```
### 4. Structured Logging with Winston
@@ -169,17 +178,19 @@ Set via `LOG_LEVEL` environment variable (default: `info`).
#### Log Output
**Console Output**: Human-readable format with colorization
```
[2024-01-01T00:00:00.000Z] INFO: Server started on port 3001
```
**File Output**: JSON format for log aggregation
```json
{
"level": "info",
"message": "Server started on port 3001",
"timestamp": "2024-01-01T00:00:00.000Z",
"service": "discord-spywatcher"
"level": "info",
"message": "Server started on port 3001",
"timestamp": "2024-01-01T00:00:00.000Z",
"service": "discord-spywatcher"
}
```
@@ -197,8 +208,8 @@ Use the `logWithRequestId` helper to include request IDs in logs:
import { logWithRequestId } from './middleware/winstonLogger';
logWithRequestId('info', 'Processing request', req.id, {
userId: user.id,
action: 'fetch_data'
userId: user.id,
action: 'fetch_data',
});
```
@@ -207,6 +218,7 @@ logWithRequestId('info', 'Processing request', req.id, {
### 1. Alerts Configuration
Set up alerts for critical metrics:
- Error rate > 5%
- Response time p95 > 2s
- Database query time > 1s
@@ -215,6 +227,7 @@ Set up alerts for critical metrics:
### 2. Dashboard Creation
Create Grafana dashboards for:
- API performance (request rate, duration, errors)
- Database performance (query duration, connection pool)
- WebSocket connections
@@ -223,6 +236,7 @@ Create Grafana dashboards for:
### 3. Log Aggregation
Configure a log aggregator to collect and analyze logs:
- ELK Stack (Elasticsearch, Logstash, Kibana)
- Grafana Loki
- Datadog Logs
@@ -231,6 +245,7 @@ Configure a log aggregator to collect and analyze logs:
### 4. Performance Monitoring
Use Sentry's performance monitoring to:
- Identify slow API endpoints
- Track database query performance
- Monitor external API calls
@@ -264,24 +279,25 @@ Use Sentry's performance monitoring to:
```yaml
version: '3.8'
services:
app:
build: .
ports:
- "3001:3001"
environment:
- SENTRY_DSN=${SENTRY_DSN}
prometheus:
image: prom/prometheus
ports:
- "9090:9090"
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
app:
build: .
ports:
- '3001:3001'
environment:
- SENTRY_DSN=${SENTRY_DSN}
prometheus:
image: prom/prometheus
ports:
- '9090:9090'
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
```
### Grafana Dashboard Import
Use the provided Prometheus metrics to create dashboards. Key panels:
- HTTP request rate (rate(http_requests_total[5m]))
- HTTP request duration (histogram_quantile(0.95, http_request_duration_seconds))
- Error rate (rate(http_requests_errors[5m]) / rate(http_requests_total[5m]))

422
PLUGIN_SYSTEM.md
View File

@@ -46,18 +46,15 @@ The manifest file contains plugin metadata and configuration:
```json
{
"id": "my-plugin",
"name": "My Plugin",
"version": "1.0.0",
"author": "Your Name",
"description": "A description of what your plugin does",
"spywatcherVersion": ">=1.0.0",
"dependencies": [],
"permissions": [
"discord:events",
"api:routes"
],
"homepage": "https://github.com/yourname/my-plugin"
"id": "my-plugin",
"name": "My Plugin",
"version": "1.0.0",
"author": "Your Name",
"description": "A description of what your plugin does",
"spywatcherVersion": ">=1.0.0",
"dependencies": [],
"permissions": ["discord:events", "api:routes"],
"homepage": "https://github.com/yourname/my-plugin"
}
```
@@ -67,49 +64,51 @@ The index file exports a plugin object implementing the `Plugin` interface:
```javascript
module.exports = {
manifest: require('./manifest.json'),
manifest: require('./manifest.json'),
// Initialize the plugin
async init(context) {
context.logger.info('Plugin initialized');
},
// Initialize the plugin
async init(context) {
context.logger.info('Plugin initialized');
},
// Start the plugin (optional)
async start() {
console.log('Plugin started');
},
// Start the plugin (optional)
async start() {
console.log('Plugin started');
},
// Stop the plugin (optional)
async stop() {
console.log('Plugin stopped');
},
// Stop the plugin (optional)
async stop() {
console.log('Plugin stopped');
},
// Clean up resources (optional)
async destroy() {
console.log('Plugin destroyed');
},
// Clean up resources (optional)
async destroy() {
console.log('Plugin destroyed');
},
// Register hooks (optional)
registerHooks(hooks) {
hooks.register('discord:messageCreate', async (message, context) => {
context.logger.info('Message received:', { content: message.content });
});
},
// Register hooks (optional)
registerHooks(hooks) {
hooks.register('discord:messageCreate', async (message, context) => {
context.logger.info('Message received:', {
content: message.content,
});
});
},
// Register API routes (optional)
registerRoutes(router) {
router.get('/hello', (req, res) => {
res.json({ message: 'Hello from plugin!' });
});
},
// Register API routes (optional)
registerRoutes(router) {
router.get('/hello', (req, res) => {
res.json({ message: 'Hello from plugin!' });
});
},
// Health check (optional)
async healthCheck() {
return {
healthy: true,
message: 'Plugin is running'
};
}
// Health check (optional)
async healthCheck() {
return {
healthy: true,
message: 'Plugin is running',
};
},
};
```
@@ -119,14 +118,14 @@ module.exports = {
```typescript
interface Plugin {
manifest: PluginManifest;
init(context: PluginContext): Promise<void> | void;
start?(): Promise<void> | void;
stop?(): Promise<void> | void;
destroy?(): Promise<void> | void;
registerHooks?(hooks: PluginHookRegistry): void;
registerRoutes?(router: Router): void;
healthCheck?(): Promise<PluginHealthStatus> | PluginHealthStatus;
manifest: PluginManifest;
init(context: PluginContext): Promise<void> | void;
start?(): Promise<void> | void;
stop?(): Promise<void> | void;
destroy?(): Promise<void> | void;
registerHooks?(hooks: PluginHookRegistry): void;
registerRoutes?(router: Router): void;
healthCheck?(): Promise<PluginHealthStatus> | PluginHealthStatus;
}
```
@@ -136,35 +135,35 @@ The plugin context provides access to services and utilities:
```typescript
interface PluginContext {
// Discord bot client (requires DISCORD_CLIENT permission)
discordClient?: Client;
// Express app (requires API_ROUTES permission)
app?: Express;
// Plugin configuration
config: Record<string, unknown>;
// Plugin data directory (for storing plugin-specific files)
dataDir: string;
// Logger for plugin messages
logger: {
info(message: string, meta?: Record<string, unknown>): void;
warn(message: string, meta?: Record<string, unknown>): void;
error(message: string, meta?: Record<string, unknown>): void;
debug(message: string, meta?: Record<string, unknown>): void;
};
// Event emitter for plugin events
events: PluginEventEmitter;
// Services (based on permissions)
services: {
database?: PrismaClient; // Requires DATABASE permission
cache?: Redis; // Requires CACHE permission
websocket?: WebSocketService; // Requires WEBSOCKET permission
};
// Discord bot client (requires DISCORD_CLIENT permission)
discordClient?: Client;
// Express app (requires API_ROUTES permission)
app?: Express;
// Plugin configuration
config: Record<string, unknown>;
// Plugin data directory (for storing plugin-specific files)
dataDir: string;
// Logger for plugin messages
logger: {
info(message: string, meta?: Record<string, unknown>): void;
warn(message: string, meta?: Record<string, unknown>): void;
error(message: string, meta?: Record<string, unknown>): void;
debug(message: string, meta?: Record<string, unknown>): void;
};
// Event emitter for plugin events
events: PluginEventEmitter;
// Services (based on permissions)
services: {
database?: PrismaClient; // Requires DATABASE permission
cache?: Redis; // Requires CACHE permission
websocket?: WebSocketService; // Requires WEBSOCKET permission
};
}
```
@@ -198,13 +197,13 @@ Plugins must declare required permissions in their manifest. Available permissio
```json
{
"permissions": [
"discord:client",
"discord:events",
"api:routes",
"database:access",
"cache:access"
]
"permissions": [
"discord:client",
"discord:events",
"api:routes",
"database:access",
"cache:access"
]
}
```
@@ -244,16 +243,16 @@ registerHooks(hooks) {
// Listen for presence updates
hooks.register('discord:presenceUpdate', async (data, context) => {
const { oldPresence, newPresence } = data;
context.logger.info('Presence updated', {
userId: newPresence.userId,
status: newPresence.status
});
// You can modify and return data to affect downstream processing
return data;
});
// Listen for new messages
hooks.register('discord:messageCreate', async (message, context) => {
if (message.content.includes('!ping')) {
@@ -270,37 +269,39 @@ registerHooks(hooks) {
A simple plugin that logs all Discord messages:
**manifest.json:**
```json
{
"id": "message-logger",
"name": "Message Logger",
"version": "1.0.0",
"author": "SpyWatcher Team",
"description": "Logs all Discord messages to a file",
"permissions": ["discord:events", "fs:access"]
"id": "message-logger",
"name": "Message Logger",
"version": "1.0.0",
"author": "SpyWatcher Team",
"description": "Logs all Discord messages to a file",
"permissions": ["discord:events", "fs:access"]
}
```
**index.js:**
```javascript
const fs = require('fs');
const path = require('path');
module.exports = {
manifest: require('./manifest.json'),
async init(context) {
this.context = context;
this.logFile = path.join(context.dataDir, 'messages.log');
context.logger.info('Message logger initialized');
},
registerHooks(hooks) {
hooks.register('discord:messageCreate', async (message, context) => {
const logEntry = `${new Date().toISOString()} - ${message.author.username}: ${message.content}\n`;
fs.appendFileSync(this.logFile, logEntry);
});
}
manifest: require('./manifest.json'),
async init(context) {
this.context = context;
this.logFile = path.join(context.dataDir, 'messages.log');
context.logger.info('Message logger initialized');
},
registerHooks(hooks) {
hooks.register('discord:messageCreate', async (message, context) => {
const logEntry = `${new Date().toISOString()} - ${message.author.username}: ${message.content}\n`;
fs.appendFileSync(this.logFile, logEntry);
});
},
};
```
@@ -309,53 +310,55 @@ module.exports = {
A plugin that adds custom analytics endpoints:
**manifest.json:**
```json
{
"id": "custom-analytics",
"name": "Custom Analytics",
"version": "1.0.0",
"author": "SpyWatcher Team",
"description": "Provides custom analytics endpoints",
"permissions": ["api:routes", "database:access"]
"id": "custom-analytics",
"name": "Custom Analytics",
"version": "1.0.0",
"author": "SpyWatcher Team",
"description": "Provides custom analytics endpoints",
"permissions": ["api:routes", "database:access"]
}
```
**index.js:**
```javascript
module.exports = {
manifest: require('./manifest.json'),
async init(context) {
this.context = context;
this.db = context.services.database;
context.logger.info('Custom analytics initialized');
},
registerRoutes(router) {
// GET /api/plugins/custom-analytics/stats
router.get('/stats', async (req, res) => {
const messageCount = await this.db.messageEvent.count();
const userCount = await this.db.user.count();
res.json({
messages: messageCount,
users: userCount,
timestamp: new Date()
});
});
// GET /api/plugins/custom-analytics/top-users
router.get('/top-users', async (req, res) => {
const topUsers = await this.db.messageEvent.groupBy({
by: ['userId'],
_count: { userId: true },
orderBy: { _count: { userId: 'desc' } },
take: 10
});
res.json({ topUsers });
});
}
manifest: require('./manifest.json'),
async init(context) {
this.context = context;
this.db = context.services.database;
context.logger.info('Custom analytics initialized');
},
registerRoutes(router) {
// GET /api/plugins/custom-analytics/stats
router.get('/stats', async (req, res) => {
const messageCount = await this.db.messageEvent.count();
const userCount = await this.db.user.count();
res.json({
messages: messageCount,
users: userCount,
timestamp: new Date(),
});
});
// GET /api/plugins/custom-analytics/top-users
router.get('/top-users', async (req, res) => {
const topUsers = await this.db.messageEvent.groupBy({
by: ['userId'],
_count: { userId: true },
orderBy: { _count: { userId: 'desc' } },
take: 10,
});
res.json({ topUsers });
});
},
};
```
@@ -364,59 +367,61 @@ module.exports = {
A plugin that sends notifications for specific events:
**manifest.json:**
```json
{
"id": "notifications",
"name": "Notification Plugin",
"version": "1.0.0",
"author": "SpyWatcher Team",
"description": "Sends notifications via webhook",
"permissions": ["discord:events", "network:access", "websocket:access"]
"id": "notifications",
"name": "Notification Plugin",
"version": "1.0.0",
"author": "SpyWatcher Team",
"description": "Sends notifications via webhook",
"permissions": ["discord:events", "network:access", "websocket:access"]
}
```
**index.js:**
```javascript
const axios = require('axios');
module.exports = {
manifest: require('./manifest.json'),
async init(context) {
this.context = context;
this.webhookUrl = process.env.NOTIFICATION_WEBHOOK_URL;
context.logger.info('Notification plugin initialized');
},
registerHooks(hooks) {
// Notify on multi-client detection
hooks.register('discord:presenceUpdate', async (data, context) => {
const { newPresence } = data;
const platforms = Object.keys(newPresence.clientStatus || {});
if (platforms.length > 1) {
await this.sendNotification({
type: 'multi-client',
user: newPresence.user.username,
platforms: platforms.join(', ')
manifest: require('./manifest.json'),
async init(context) {
this.context = context;
this.webhookUrl = process.env.NOTIFICATION_WEBHOOK_URL;
context.logger.info('Notification plugin initialized');
},
registerHooks(hooks) {
// Notify on multi-client detection
hooks.register('discord:presenceUpdate', async (data, context) => {
const { newPresence } = data;
const platforms = Object.keys(newPresence.clientStatus || {});
if (platforms.length > 1) {
await this.sendNotification({
type: 'multi-client',
user: newPresence.user.username,
platforms: platforms.join(', '),
});
}
return data;
});
}
return data;
});
},
async sendNotification(data) {
if (!this.webhookUrl) return;
try {
await axios.post(this.webhookUrl, {
text: `🔔 ${data.type}: ${data.user} detected on ${data.platforms}`
});
} catch (error) {
this.context.logger.error('Failed to send notification', { error });
}
}
},
async sendNotification(data) {
if (!this.webhookUrl) return;
try {
await axios.post(this.webhookUrl, {
text: `🔔 ${data.type}: ${data.user} detected on ${data.platforms}`,
});
} catch (error) {
this.context.logger.error('Failed to send notification', { error });
}
},
};
```
@@ -499,9 +504,9 @@ Request only the permissions you need:
```json
{
"permissions": [
"discord:events" // Only request what's necessary
]
"permissions": [
"discord:events" // Only request what's necessary
]
}
```
@@ -513,11 +518,11 @@ Always validate external data:
registerRoutes(router) {
router.post('/data', (req, res) => {
const { value } = req.body;
if (!value || typeof value !== 'string') {
return res.status(400).json({ error: 'Invalid input' });
}
// Process validated data
});
}
@@ -536,7 +541,7 @@ registerRoutes(router) {
windowMs: 15 * 60 * 1000,
max: 100
});
router.use(limiter);
}
```
@@ -567,11 +572,11 @@ UNINITIALIZED → INITIALIZING → INITIALIZED → STARTING → RUNNING
const plugin = require('../index.js');
describe('My Plugin', () => {
it('should initialize', async () => {
const context = createMockContext();
await plugin.init(context);
expect(context.logger.info).toHaveBeenCalledWith('Plugin initialized');
});
it('should initialize', async () => {
const context = createMockContext();
await plugin.init(context);
expect(context.logger.info).toHaveBeenCalledWith('Plugin initialized');
});
});
```
@@ -608,6 +613,7 @@ npm test -- plugins/my-plugin
## API Reference
For complete API reference, see:
- [Plugin Types](./backend/src/plugins/types.ts)
- [Plugin Loader](./backend/src/plugins/PluginLoader.ts)
- [Plugin Manager](./backend/src/plugins/PluginManager.ts)
@@ -615,12 +621,14 @@ For complete API reference, see:
## Support
For questions and support:
- GitHub Issues: [discord-spywatcher/issues](https://github.com/subculture-collective/discord-spywatcher/issues)
- Documentation: [README.md](./README.md)
## Contributing
To contribute a plugin:
1. Create your plugin following this guide
2. Test thoroughly
3. Document usage and configuration

154
POSTGRESQL.md
View File

@@ -3,6 +3,7 @@
This guide covers the PostgreSQL configuration, features, and management for Discord SpyWatcher.
## Table of Contents
- [Overview](#overview)
- [PostgreSQL-Specific Features](#postgresql-specific-features)
- [Connection Configuration](#connection-configuration)
@@ -15,6 +16,7 @@ This guide covers the PostgreSQL configuration, features, and management for Dis
## Overview
Discord SpyWatcher uses PostgreSQL 15+ as its production database, providing:
- Advanced data types (JSONB, arrays, UUIDs, timestamps with timezone)
- Full-text search capabilities
- Better concurrency and performance
@@ -26,54 +28,58 @@ Discord SpyWatcher uses PostgreSQL 15+ as its production database, providing:
### Advanced Data Types
#### JSONB Fields
All event models include a `metadata` field using JSONB for flexible, queryable JSON storage:
```typescript
// Store flexible metadata
await db.presenceEvent.create({
data: {
userId: "123",
username: "user",
clients: ["desktop", "mobile"],
metadata: {
status: "online",
activities: ["gaming"],
customField: "value"
}
}
data: {
userId: '123',
username: 'user',
clients: ['desktop', 'mobile'],
metadata: {
status: 'online',
activities: ['gaming'],
customField: 'value',
},
},
});
// Query JSONB data
const events = await db.presenceEvent.findMany({
where: {
metadata: {
path: ['status'],
equals: 'online'
}
}
where: {
metadata: {
path: ['status'],
equals: 'online',
},
},
});
```
#### Array Fields
Comma-separated strings have been converted to native PostgreSQL arrays:
```typescript
// PresenceEvent.clients - array of client types
clients: ["desktop", "web", "mobile"]
clients: ['desktop', 'web', 'mobile'];
// RoleChangeEvent.addedRoles - array of role IDs
addedRoles: ["123456789", "987654321"]
addedRoles: ['123456789', '987654321'];
```
#### UUID Primary Keys
Event models use UUIDs for better distribution and security:
```typescript
// Auto-generated UUID
id: "550e8400-e29b-41d4-a716-446655440000"
id: '550e8400-e29b-41d4-a716-446655440000';
```
#### Timezone-Aware Timestamps
All timestamp fields use `TIMESTAMPTZ` for proper timezone handling:
```typescript
@@ -93,6 +99,7 @@ DB_PASSWORD=yourpassword ./scripts/setup-fulltext-search.sh
```
This creates:
- A generated `content_search` tsvector column
- A GIN index for efficient text searches
@@ -122,6 +129,7 @@ const results = await db.$queryRaw`
```
#### Search Operators
- `&` - AND (both terms must be present)
- `|` - OR (either term can be present)
- `!` - NOT (term must not be present)
@@ -132,6 +140,7 @@ Example: `'cat & dog'` finds messages with both "cat" and "dog"
### Performance Features
#### Optimized Indexes
The schema includes strategic indexes for common queries:
```prisma
@@ -152,6 +161,7 @@ The schema includes strategic indexes for common queries:
```
#### Composite Indexes
Some models use composite indexes for complex queries:
```prisma
@@ -169,26 +179,29 @@ DATABASE_URL="postgresql://username:password@host:port/database?schema=public&co
### Connection Pooling Parameters
| Parameter | Recommended Value | Description |
|-----------|-------------------|-------------|
| `connection_limit` | 10-50 | Maximum number of connections in the pool |
| `pool_timeout` | 20 | Seconds to wait for an available connection |
| `connect_timeout` | 10 | Seconds to wait for initial connection |
| `sslmode` | require (prod) | SSL/TLS encryption mode |
| Parameter | Recommended Value | Description |
| ------------------ | ----------------- | ------------------------------------------- |
| `connection_limit` | 10-50 | Maximum number of connections in the pool |
| `pool_timeout` | 20 | Seconds to wait for an available connection |
| `connect_timeout` | 10 | Seconds to wait for initial connection |
| `sslmode` | require (prod) | SSL/TLS encryption mode |
### Example Configurations
#### Development
```bash
DATABASE_URL="postgresql://spywatcher:password@localhost:5432/spywatcher?connection_limit=10&pool_timeout=20"
```
#### Production
```bash
DATABASE_URL="postgresql://spywatcher:securepassword@db.example.com:5432/spywatcher?sslmode=require&connection_limit=50&pool_timeout=20&connect_timeout=10"
```
#### Docker
```bash
DATABASE_URL="postgresql://spywatcher:${DB_PASSWORD}@postgres:5432/spywatcher?connection_limit=20&pool_timeout=20"
```
@@ -198,17 +211,20 @@ DATABASE_URL="postgresql://spywatcher:${DB_PASSWORD}@postgres:5432/spywatcher?co
### Migrations
#### Create a Migration
```bash
cd backend
npx prisma migrate dev --name add_new_feature
```
#### Apply Migrations (Production)
```bash
npx prisma migrate deploy
```
#### Reset Database (Development Only)
```bash
npx prisma migrate reset
```
@@ -264,36 +280,39 @@ command:
### Query Optimization
#### Use Indexes Effectively
```typescript
// Good - uses index
const events = await db.presenceEvent.findMany({
where: { userId: "123" },
orderBy: { createdAt: 'desc' }
where: { userId: '123' },
orderBy: { createdAt: 'desc' },
});
// Bad - no index on username alone
const events = await db.presenceEvent.findMany({
where: { username: "john" }
where: { username: 'john' },
});
```
#### Batch Operations
```typescript
// Use createMany for bulk inserts
await db.presenceEvent.createMany({
data: events,
skipDuplicates: true
data: events,
skipDuplicates: true,
});
```
#### Pagination
```typescript
// Efficient pagination with cursor
const events = await db.presenceEvent.findMany({
take: 20,
skip: 1,
cursor: { id: lastId },
orderBy: { createdAt: 'desc' }
take: 20,
skip: 1,
cursor: { id: lastId },
orderBy: { createdAt: 'desc' },
});
```
@@ -303,7 +322,7 @@ Enable query logging in development:
```typescript
const db = new PrismaClient({
log: ['query', 'error', 'warn'],
log: ['query', 'error', 'warn'],
});
```
@@ -322,6 +341,7 @@ DB_PASSWORD=yourpassword npm run db:backup
```
### Backup Features
- Compressed backups (gzip)
- 30-day retention by default
- Optional S3 upload
@@ -349,6 +369,7 @@ ALTER SYSTEM SET archive_command = 'cp %p /path/to/archive/%f';
### Database Metrics
#### Connection Count
```sql
SELECT count(*), state
FROM pg_stat_activity
@@ -357,11 +378,13 @@ GROUP BY state;
```
#### Database Size
```sql
SELECT pg_size_pretty(pg_database_size('spywatcher'));
```
#### Table Sizes
```sql
SELECT
schemaname,
@@ -373,6 +396,7 @@ ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC;
```
#### Index Usage
```sql
SELECT
schemaname || '.' || tablename AS table,
@@ -385,6 +409,7 @@ ORDER BY idx_scan DESC;
```
#### Slow Queries
```sql
SELECT
pid,
@@ -407,6 +432,7 @@ DB_PASSWORD=yourpassword npm run db:maintenance
```
This performs:
- VACUUM ANALYZE (cleanup and optimization)
- Statistics updates
- Bloat detection
@@ -417,6 +443,7 @@ This performs:
### Performance Monitoring Tools
#### pg_stat_statements
Enable query statistics:
```sql
@@ -439,22 +466,26 @@ LIMIT 10;
### Connection Issues
#### Problem: Cannot connect to database
```
Error: P1001: Can't reach database server at `postgres:5432`
```
**Solutions:**
- Check if PostgreSQL is running: `docker-compose ps`
- Verify DATABASE_URL is correct
- Check network connectivity
- Ensure PostgreSQL container is healthy
#### Problem: Too many connections
```
Error: FATAL: remaining connection slots are reserved
```
**Solutions:**
- Increase `max_connections` in PostgreSQL configuration
- Reduce `connection_limit` in DATABASE_URL
- Check for connection leaks in application code
@@ -465,6 +496,7 @@ Error: FATAL: remaining connection slots are reserved
#### Problem: Slow queries
**Solutions:**
- Enable query logging to identify slow queries
- Add indexes for commonly queried fields
- Use EXPLAIN ANALYZE to understand query plans
@@ -474,6 +506,7 @@ Error: FATAL: remaining connection slots are reserved
#### Problem: High memory usage
**Solutions:**
- Reduce `shared_buffers` if too high
- Adjust `work_mem` for complex queries
- Run VACUUM to reclaim space
@@ -484,6 +517,7 @@ Error: FATAL: remaining connection slots are reserved
#### Problem: Migration fails with schema mismatch
**Solutions:**
```bash
# Check migration status
npx prisma migrate status
@@ -498,6 +532,7 @@ npx prisma migrate reset
#### Problem: Type errors after migration
**Solutions:**
```bash
# Regenerate Prisma Client
npx prisma generate
@@ -511,6 +546,7 @@ npm run build
#### Problem: UUID vs Integer ID conflicts
**Solution:** Use the migration script to handle ID conversion:
```bash
npm run db:migrate
```
@@ -518,6 +554,7 @@ npm run db:migrate
#### Problem: Array field errors
**Solution:** Ensure comma-separated strings are converted to arrays:
```typescript
// Old: clients: "desktop,mobile"
// New: clients: ["desktop", "mobile"]
@@ -526,39 +563,39 @@ npm run db:migrate
## Best Practices
1. **Connection Management**
- Always use connection pooling
- Close connections when done
- Set appropriate pool limits
- Always use connection pooling
- Close connections when done
- Set appropriate pool limits
2. **Indexing**
- Index foreign keys
- Index frequently queried fields
- Monitor index usage
- Remove unused indexes
- Index foreign keys
- Index frequently queried fields
- Monitor index usage
- Remove unused indexes
3. **Query Optimization**
- Use prepared statements
- Avoid N+1 queries
- Use batch operations
- Implement pagination
- Use prepared statements
- Avoid N+1 queries
- Use batch operations
- Implement pagination
4. **Security**
- Use SSL/TLS in production
- Rotate credentials regularly
- Limit user permissions
- Enable audit logging
- Use SSL/TLS in production
- Rotate credentials regularly
- Limit user permissions
- Enable audit logging
5. **Backup and Recovery**
- Automate backups
- Test restore procedures
- Store backups securely
- Document recovery process
- Automate backups
- Test restore procedures
- Store backups securely
- Document recovery process
6. **Monitoring**
- Track query performance
- Monitor connection usage
- Watch for slow queries
- Set up alerts
- Track query performance
- Monitor connection usage
- Watch for slow queries
- Set up alerts
## Additional Resources
@@ -570,6 +607,7 @@ npm run db:migrate
## Support
For issues or questions:
- Check the [main README](../README.md)
- Review [MIGRATION.md](../MIGRATION.md)
- Review [scripts/README.md](../scripts/README.md)

92
POSTGRESQL_MIGRATION_SUMMARY.md
View File

@@ -7,6 +7,7 @@ This document summarizes the PostgreSQL migration implementation for Discord Spy
All requirements from the original issue have been implemented:
### ✅ PostgreSQL Setup
- [x] PostgreSQL 15+ configuration in Docker Compose
- [x] Optimal configuration for workload (tuned parameters)
- [x] Connection pooling setup (via DATABASE_URL parameters)
@@ -15,6 +16,7 @@ All requirements from the original issue have been implemented:
- [x] Replication setup (documented for production)
### ✅ Schema Migration
- [x] Prisma datasource updated to PostgreSQL
- [x] Data type optimization (JSONB, arrays, UUIDs, TIMESTAMPTZ)
- [x] Index strategy review and optimization
@@ -22,6 +24,7 @@ All requirements from the original issue have been implemented:
- [x] Schema validates successfully
### ✅ Data Migration
- [x] Automated migration script from SQLite
- [x] Data transformation (IDs to UUIDs, strings to arrays)
- [x] Batch processing with progress tracking
@@ -32,22 +35,26 @@ All requirements from the original issue have been implemented:
### ✅ PostgreSQL-Specific Features
#### Advanced Data Types
- [x] JSONB for flexible metadata storage
- [x] Array types for multi-value fields (clients, roles)
- [x] UUID for primary keys in event models
- [x] TIMESTAMPTZ for timezone-aware timestamps
#### Full-Text Search
- [x] PostgreSQL FTS setup script
- [x] GIN indexes for search performance
- [x] Query examples and documentation
#### Advanced Queries
- [x] Documentation for window functions, CTEs
- [x] JSONB query examples
- [x] Optimized indexes for common patterns
#### Performance Features
- [x] Strategic indexes on all models
- [x] Composite indexes for complex queries
- [x] Optimized PostgreSQL configuration
@@ -55,12 +62,14 @@ All requirements from the original issue have been implemented:
### ✅ Database Management
#### Backup Strategy
- [x] Automated backup script (pg_dump with compression)
- [x] Configurable retention policy (default 30 days)
- [x] Optional S3 upload support
- [x] Backup verification
#### Monitoring
- [x] Maintenance script with monitoring queries
- [x] Query performance monitoring
- [x] Connection monitoring
@@ -69,6 +78,7 @@ All requirements from the original issue have been implemented:
- [x] Database size tracking
#### Maintenance
- [x] Automated VACUUM/ANALYZE script
- [x] Index usage analysis
- [x] Long-running query detection
@@ -77,6 +87,7 @@ All requirements from the original issue have been implemented:
## 📁 Files Created/Modified
### Schema and Configuration
- `backend/prisma/schema.prisma` - Enhanced with PostgreSQL features
- `backend/src/db.ts` - Connection pooling and singleton pattern
- `backend/package.json` - Added database management scripts
@@ -84,6 +95,7 @@ All requirements from the original issue have been implemented:
- `docker-compose.prod.yml` - Production PostgreSQL configuration
### Management Scripts (scripts/)
- `postgres-init.sql` - Database initialization with extensions
- `backup.sh` - Automated backup with retention
- `restore.sh` - Interactive restore with verification
@@ -94,6 +106,7 @@ All requirements from the original issue have been implemented:
- `README.md` - Complete script documentation
### Documentation
- `POSTGRESQL.md` - Complete PostgreSQL feature guide (12KB)
- `MIGRATION.md` - Updated migration guide (significant rewrite)
- `scripts/README.md` - Script usage documentation
@@ -103,6 +116,7 @@ All requirements from the original issue have been implemented:
### 1. Production-Ready PostgreSQL Configuration
**Optimized Parameters:**
```yaml
max_connections: 100
shared_buffers: 256MB
@@ -116,6 +130,7 @@ work_mem: 4MB
### 2. Advanced Data Types
**Before (SQLite):**
```prisma
model PresenceEvent {
id Int @id @default(autoincrement())
@@ -125,13 +140,14 @@ model PresenceEvent {
```
**After (PostgreSQL):**
```prisma
model PresenceEvent {
id String @id @default(uuid())
clients String[] // ["desktop", "web"]
metadata Json? @db.JsonB
createdAt DateTime @default(now()) @db.Timestamptz
@@index([userId])
@@index([createdAt])
}
@@ -155,16 +171,19 @@ const results = await db.$queryRaw`
### 4. Automated Management
**Backup:**
```bash
DB_PASSWORD=pass npm run db:backup
```
**Maintenance:**
```bash
DB_PASSWORD=pass npm run db:maintenance
```
**Migration:**
```bash
npm run db:migrate:dry # Test first
npm run db:migrate # Actual migration
@@ -173,17 +192,19 @@ npm run db:migrate # Actual migration
## 📊 Schema Changes Summary
### Event Models (Breaking Changes)
| Model | ID Type | Array Fields | Metadata | Timestamps |
|-------|---------|--------------|----------|------------|
| PresenceEvent | Int → UUID | clients → String[] | Added JSONB | → Timestamptz |
| TypingEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| MessageEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| JoinEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| DeletedMessageEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| ReactionTime | Int → UUID | - | Added JSONB | → Timestamptz |
| RoleChangeEvent | Int → UUID | addedRoles → String[] | Added JSONB | → Timestamptz |
| Model | ID Type | Array Fields | Metadata | Timestamps |
| ------------------- | ---------- | --------------------- | ----------- | ------------- |
| PresenceEvent | Int → UUID | clients → String[] | Added JSONB | → Timestamptz |
| TypingEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| MessageEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| JoinEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| DeletedMessageEvent | Int → UUID | - | Added JSONB | → Timestamptz |
| ReactionTime | Int → UUID | - | Added JSONB | → Timestamptz |
| RoleChangeEvent | Int → UUID | addedRoles → String[] | Added JSONB | → Timestamptz |
### All Models
- All timestamps upgraded to `@db.Timestamptz`
- All JSON fields upgraded to `@db.JsonB`
- Strategic indexes added for performance
@@ -246,17 +267,20 @@ docker-compose -f docker-compose.prod.yml exec backend sh -c "DB_PASSWORD=$DB_PA
## 📈 Performance Improvements
### Indexes
- Strategic indexes on userId, guildId, channelId, createdAt
- Composite indexes for common query patterns
- GIN indexes for full-text search
- Optimized for both read and write operations
### Connection Pooling
```bash
DATABASE_URL="postgresql://user:pass@host:5432/db?connection_limit=20&pool_timeout=20"
```
### Optimized Queries
- Batch operations with createMany
- Cursor-based pagination
- JSONB field queries
@@ -273,22 +297,22 @@ DATABASE_URL="postgresql://user:pass@host:5432/db?connection_limit=20&pool_timeo
## 📚 Documentation Structure
1. **POSTGRESQL.md** (12KB)
- Complete PostgreSQL feature guide
- Connection configuration
- Performance optimization
- Monitoring and troubleshooting
- Complete PostgreSQL feature guide
- Connection configuration
- Performance optimization
- Monitoring and troubleshooting
2. **MIGRATION.md** (Updated)
- Step-by-step migration guide
- Schema change documentation
- Post-migration steps
- Troubleshooting guide
- Step-by-step migration guide
- Schema change documentation
- Post-migration steps
- Troubleshooting guide
3. **scripts/README.md**
- Script usage documentation
- Environment variables
- Automation examples
- Best practices
- Script usage documentation
- Environment variables
- Automation examples
- Best practices
## ✅ Testing Status
@@ -313,27 +337,28 @@ DATABASE_URL="postgresql://user:pass@host:5432/db?connection_limit=20&pool_timeo
These are beyond the original requirements but could be added:
1. **Monitoring Dashboard**
- Grafana + Prometheus for metrics
- Custom dashboards for database health
- Grafana + Prometheus for metrics
- Custom dashboards for database health
2. **Replication**
- Primary-replica setup
- Streaming replication configuration
- Automatic failover
- Primary-replica setup
- Streaming replication configuration
- Automatic failover
3. **Advanced Analytics**
- Materialized views for dashboards
- Window functions for time-series analysis
- Aggregate functions for statistics
- Materialized views for dashboards
- Window functions for time-series analysis
- Aggregate functions for statistics
4. **Partitioning**
- Table partitioning for large tables
- Time-based partitioning for events
- Automatic partition management
- Table partitioning for large tables
- Time-based partitioning for events
- Automatic partition management
## 📞 Support
For questions or issues:
- Review [POSTGRESQL.md](./POSTGRESQL.md)
- Review [MIGRATION.md](./MIGRATION.md)
- Review [scripts/README.md](./scripts/README.md)
@@ -343,6 +368,7 @@ For questions or issues:
## 🏆 Summary
This implementation provides a production-ready PostgreSQL setup with:
- Advanced PostgreSQL features (JSONB, arrays, UUIDs, full-text search)
- Automated management (backup, restore, maintenance)
- Comprehensive documentation

66
PRIVACY_ANALYTICS.md
View File

@@ -7,22 +7,26 @@
Discord SpyWatcher collects usage analytics to improve the application and understand how users interact with our features. The following data may be collected:
#### With Your Consent (Opt-in)
When you accept analytics tracking:
- **User Identifier**: Your user ID for correlating events
- **Session Information**: Session IDs to track user journeys
- **Usage Data**:
- Pages you visit
- Features you use
- Buttons you click
- Forms you submit
- Pages you visit
- Features you use
- Buttons you click
- Forms you submit
- **Technical Data**:
- IP address (for geographic insights)
- Browser user agent
- Referrer URLs
- Response times and performance metrics
- IP address (for geographic insights)
- Browser user agent
- Referrer URLs
- Response times and performance metrics
#### Without Consent (Anonymized)
When you decline or haven't provided consent:
- **Anonymized Events**: All events are tracked but personal identifiers are hashed
- **Performance Metrics**: Aggregated response times and system performance
- **Error Events**: Anonymized error tracking for debugging
@@ -30,6 +34,7 @@ When you decline or haven't provided consent:
### What We Don't Collect
We DO NOT collect:
- Message content
- Private conversations
- Passwords or credentials
@@ -40,6 +45,7 @@ We DO NOT collect:
## How We Use Analytics
### Primary Purposes
1. **Feature Improvement**: Understand which features are most valuable
2. **Performance Optimization**: Identify and fix slow endpoints
3. **Error Detection**: Catch and resolve bugs faster
@@ -47,6 +53,7 @@ We DO NOT collect:
5. **Capacity Planning**: Understand usage patterns for scaling
### Secondary Purposes
- Generating aggregated, anonymized statistics
- Creating usage reports for transparency
- Research and development
@@ -54,6 +61,7 @@ We DO NOT collect:
## Your Rights & Choices
### Consent Management
You have complete control over analytics tracking:
**Opt-In**: Accept the consent banner to help improve the application
@@ -61,6 +69,7 @@ You have complete control over analytics tracking:
**Change Consent**: Access Settings > Privacy to update your preferences
### Your Rights Under GDPR
If you are in the European Union, you have the right to:
1. **Right to Access**: Request a copy of your analytics data
@@ -72,7 +81,9 @@ If you are in the European Union, you have the right to:
7. **Right to Withdraw Consent**: Change your consent status at any time
### Exercising Your Rights
To exercise any of these rights:
1. Navigate to Settings > Privacy in the application
2. Use the data export feature
3. Submit a data deletion request
@@ -81,19 +92,24 @@ To exercise any of these rights:
## Data Storage & Security
### Where Data is Stored
Analytics data is stored:
- In our secure PostgreSQL database
- Encrypted at rest
- Accessible only to authorized personnel
- Located in [specify region/provider]
### How Long We Keep Data
- **Raw Analytics Events**: 90 days by default (configurable)
- **Aggregated Summaries**: Indefinitely (anonymized)
- **Deleted Account Data**: Removed within 30 days
### Security Measures
We protect your data with:
- Encryption in transit (HTTPS/TLS)
- Encryption at rest
- Access controls and authentication
@@ -104,18 +120,22 @@ We protect your data with:
## Data Sharing & Third Parties
### We DO NOT:
- Sell your analytics data to third parties
- Share personal data with advertisers
- Use your data for marketing without consent
- Transfer data outside our processing purposes
### We MAY Share:
- Anonymized, aggregated statistics publicly
- Data with service providers (hosting, monitoring)
- Data when legally required (court orders, etc.)
### Service Providers
We use the following services that may have access to aggregated data:
- **Hosting Provider**: [Provider name] - Data hosting
- **Monitoring Service**: Sentry - Error tracking
- **Metrics Service**: Prometheus - Performance monitoring
@@ -125,6 +145,7 @@ All service providers are contractually bound to protect your data.
## Anonymization Technology
### How We Anonymize
When you decline consent or before data retention expires:
1. **Hashing**: Personal identifiers are converted using SHA-256 cryptographic hash
@@ -133,12 +154,14 @@ When you decline consent or before data retention expires:
4. **Consistent**: Same input produces same hash for analytics correlation
### Example
```
Original: user-id-12345
Hashed: 8d969eef6ecad3c2
```
### What Remains Anonymous
- Event types (page views, clicks)
- Feature usage patterns
- Performance metrics
@@ -150,25 +173,31 @@ Hashed: 8d969eef6ecad3c2
### Cookies We Use
**Analytics Consent Cookie**
- Name: `analyticsConsent`
- Purpose: Remember your consent choice
- Duration: 1 year
- Type: First-party, necessary for consent management
**Session Cookie**
- Name: [session cookie name]
- Purpose: Maintain your login session
- Duration: Session
- Type: First-party, necessary for authentication
### Local Storage
We use browser local storage for:
- Analytics consent preferences
- User interface preferences
- Temporary caching
### No Third-Party Trackers
We do not use:
- Google Analytics
- Facebook Pixel
- Advertising networks
@@ -179,6 +208,7 @@ We do not use:
Discord SpyWatcher is not intended for users under 13 years of age. We do not knowingly collect analytics data from children under 13.
If you believe we have collected data from a child under 13:
1. Contact us immediately
2. Provide the relevant user information
3. We will delete the data within 24 hours
@@ -186,6 +216,7 @@ If you believe we have collected data from a child under 13:
## International Data Transfers
If you access our service from outside [primary region]:
- Your data may be transferred to and processed in [region]
- We comply with GDPR for EU users
- Standard contractual clauses are used for data transfers
@@ -194,13 +225,16 @@ If you access our service from outside [primary region]:
## Changes to Privacy Policy
### How We Notify Changes
- Email notification to registered users
- In-app notification banner
- Updated "Last Modified" date on this page
- Consent re-request for material changes
### Your Options
When we make material changes:
1. Review the updated policy
2. Accept or decline the new terms
3. Contact us with questions
@@ -209,6 +243,7 @@ When we make material changes:
## Contact Information
### Privacy Questions
For questions about this privacy policy or our analytics practices:
**Email**: [privacy@example.com]
@@ -216,7 +251,9 @@ For questions about this privacy policy or our analytics practices:
**Address**: [physical address if applicable]
### Response Time
We aim to respond to privacy inquiries within:
- General questions: 7 business days
- Data access requests: 30 days
- Data deletion requests: 30 days
@@ -225,13 +262,16 @@ We aim to respond to privacy inquiries within:
## Compliance
### Standards We Follow
- **GDPR**: General Data Protection Regulation (EU)
- **CCPA**: California Consumer Privacy Act
- **Privacy Shield**: [If applicable]
- **Industry Best Practices**: OWASP, NIST frameworks
### Regular Audits
We conduct:
- Annual privacy policy reviews
- Quarterly security assessments
- Regular data retention cleanups
@@ -240,21 +280,27 @@ We conduct:
## Transparency
### Analytics Dashboard
View aggregated, anonymized analytics:
- Navigate to `/metrics` in the application
- Requires authentication
- Shows anonymized usage patterns
- No personal data exposed
### Data Access
Request your personal analytics data:
1. Go to Settings > Privacy
2. Click "Request Data Export"
3. Receive download link within 30 days
4. Data provided in JSON format
### Public Reports
We may publish:
- Quarterly usage statistics (anonymized)
- Feature adoption reports
- Performance benchmarks
@@ -263,14 +309,18 @@ We may publish:
## Best Practices for Users
### Maximize Privacy
To minimize data collection:
1. Decline analytics consent
2. Use private browsing mode
3. Clear cookies regularly
4. Review privacy settings periodically
### Report Concerns
If you believe your privacy has been violated:
1. Contact us immediately
2. Provide specific details
3. We investigate within 48 hours

156
QUERY_OPTIMIZATIONS.md
View File

@@ -11,6 +11,7 @@ All analytics queries have been optimized to use database-level aggregation inst
### 1. Ghost Detection (`src/analytics/ghosts.ts`)
**Original Implementation:**
```typescript
// Two separate groupBy queries
const typings = await db.typingEvent.groupBy({ ... });
@@ -19,9 +20,10 @@ const messages = await db.messageEvent.groupBy({ ... });
```
**Optimized Implementation:**
```sql
-- Single query with FULL OUTER JOIN
SELECT
SELECT
COALESCE(t."userId", m."userId") as "userId",
COALESCE(t.username, m.username) as username,
COALESCE(t.typing_count, 0) as typing_count,
@@ -45,6 +47,7 @@ LIMIT 100
### 2. Lurker Detection (`src/analytics/lurkers.ts`)
**Original Implementation:**
```typescript
// Three separate findMany calls
const presence = await db.presenceEvent.findMany({ ... });
@@ -54,6 +57,7 @@ const messages = await db.messageEvent.findMany({ ... });
```
**Optimized Implementation:**
```sql
-- Single query with LEFT JOIN and UNION
SELECT p."userId", p.username, ...
@@ -79,14 +83,16 @@ WHERE COALESCE(a.activity_count, 0) = 0
### 3. Reaction Stats (`src/analytics/reactions.ts`)
**Original Implementation:**
```typescript
const reactions = await db.reactionTime.findMany({ ... });
// In-memory aggregation with Map
```
**Optimized Implementation:**
```sql
SELECT
SELECT
"observerId" as "userId",
MAX("observerName") as username,
AVG("deltaMs")::float as avg_reaction_time,
@@ -104,14 +110,16 @@ ORDER BY avg_reaction_time ASC
### 4. Channel Diversity (`src/analytics/channels.ts`)
**Original Implementation:**
```typescript
const events = await db.typingEvent.findMany({ ... });
// Build Map with Set for unique channels per user
```
**Optimized Implementation:**
```sql
SELECT
SELECT
"userId",
MAX("username") as username,
COUNT(DISTINCT "channelId") as channel_count
@@ -129,14 +137,16 @@ LIMIT 100
### 5. Multi-Client Login Counts (`src/analytics/presence.ts`)
**Original Implementation:**
```typescript
const events = await db.presenceEvent.findMany({ ... });
// Filter events with 2+ clients in memory
```
**Optimized Implementation:**
```sql
SELECT
SELECT
"userId",
MAX("username") as username,
COUNT(*) as multi_client_count
@@ -155,14 +165,16 @@ LIMIT 100
### 6. Role Drift Detection (`src/analytics/roles.ts`)
**Original Implementation:**
```typescript
const events = await db.roleChangeEvent.findMany({ ... });
// Aggregate in Map
```
**Optimized Implementation:**
```sql
SELECT
SELECT
"userId",
MAX("username") as username,
COUNT(*) as role_change_count
@@ -179,6 +191,7 @@ LIMIT 100
### 7. Behavior Shift Detection (`src/analytics/shifts.ts`)
**Original Implementation:**
```typescript
// FOUR separate findMany queries
const pastMessages = await db.messageEvent.findMany({ ... });
@@ -189,6 +202,7 @@ const recentTyping = await db.typingEvent.findMany({ ... });
```
**Optimized Implementation:**
```sql
-- Single query with 4 CTEs
WITH past_messages AS (
@@ -232,18 +246,19 @@ Provides consistent pagination across all API endpoints:
### Paginated Endpoints
1. **Audit Logs** (`GET /api/admin/privacy/audit-logs`)
- Query params: `?page=1&limit=50`
- Returns: `{ data: [], pagination: { total, page, limit, totalPages, hasNextPage, hasPreviousPage } }`
- Query params: `?page=1&limit=50`
- Returns: `{ data: [], pagination: { total, page, limit, totalPages, hasNextPage, hasPreviousPage } }`
2. **Slow Queries** (`GET /api/admin/monitoring/database/slow-queries`)
- Query params: `?limit=20&offset=0`
- Returns: `{ data: [], pagination: { total, limit, offset }, stats: {} }`
- Query params: `?limit=20&offset=0`
- Returns: `{ data: [], pagination: { total, limit, offset }, stats: {} }`
## Slow Query Monitoring Enhancements
### Enhanced Tracking (`src/middleware/slowQueryLogger.ts`)
Added features:
- **Query text tracking** for raw SQL queries
- **Rows affected/returned** tracking
- **Pagination support** for query logs with `limit` and `offset`
@@ -252,6 +267,7 @@ Added features:
### Configuration
Environment variables:
```bash
SLOW_QUERY_THRESHOLD_MS=100 # Warning threshold
CRITICAL_QUERY_THRESHOLD_MS=1000 # Critical threshold
@@ -274,38 +290,42 @@ const result = await trackQueryPerformance(
All optimized queries leverage existing indexes:
### Composite Indexes (from Prisma schema)
- `PresenceEvent`: `(userId, createdAt DESC)`, `(userId)`, `(createdAt)`
- `MessageEvent`: `(userId, createdAt DESC)`, `(guildId, channelId)`, `(guildId, createdAt DESC)`
- `TypingEvent`: `(userId, channelId)`, `(guildId, createdAt DESC)`
- `ReactionTime`: `(observerId, createdAt DESC)`, `(guildId, createdAt DESC)`, `(deltaMs)`
### Partial Indexes (from `scripts/add-performance-indexes.sql`)
- `idx_presence_multi_client`: Only rows with `array_length(clients, 1) > 1`
- `idx_reaction_fast_delta`: Only rows with `deltaMs < 5000`
- `idx_message_recent`: Only last 90 days
- `idx_typing_recent`: Only last 90 days
### GIN Indexes
- All metadata JSONB columns have GIN indexes for efficient JSON queries
## Redis Caching Strategy
All analytics functions use Redis caching:
| Function | TTL | Reason |
|----------|-----|--------|
| Ghost Scores | 5 min | Moderately volatile data |
| Lurkers | 5 min | Moderately volatile data |
| Reaction Stats | No cache | Real-time data needed |
| Channels | 5 min | Moderately volatile data |
| Multi-Client | 5 min | Moderately volatile data |
| Role Drift | 10 min | Slower changing data |
| Behavior Shifts | 5 min | Moderately volatile data |
| Client Drift | 2 min | Rapidly changing data |
| Function | TTL | Reason |
| --------------- | -------- | ------------------------ |
| Ghost Scores | 5 min | Moderately volatile data |
| Lurkers | 5 min | Moderately volatile data |
| Reaction Stats | No cache | Real-time data needed |
| Channels | 5 min | Moderately volatile data |
| Multi-Client | 5 min | Moderately volatile data |
| Role Drift | 10 min | Slower changing data |
| Behavior Shifts | 5 min | Moderately volatile data |
| Client Drift | 2 min | Rapidly changing data |
### Cache Invalidation
Cache keys include:
- Guild ID
- Query parameters (e.g., `since` timestamp)
- Cache keys are tagged for bulk invalidation if needed
@@ -317,6 +337,7 @@ Example: `analytics:ghosts:guild123:1704067200000`
### Query Time Targets
From issue requirements:
- ✅ All queries under 100ms (p95)
- ✅ Critical queries under 50ms (p95)
- ✅ No N+1 query problems
@@ -324,17 +345,17 @@ From issue requirements:
### Measured Improvements
| Analytics Function | Before | After | Improvement |
|-------------------|--------|-------|-------------|
| Ghost Detection | ~350ms | ~100ms | 71% faster |
| Lurker Detection | ~400ms | ~100ms | 75% faster |
| Channel Diversity | ~250ms | ~70ms | 72% faster |
| Multi-Client Logins | ~200ms | ~80ms | 60% faster |
| Role Drift | ~180ms | ~90ms | 50% faster |
| Behavior Shifts | ~500ms | ~120ms | 76% faster |
| Reaction Stats | ~220ms | ~85ms | 61% faster |
| Analytics Function | Before | After | Improvement |
| ------------------- | ------ | ------ | ----------- |
| Ghost Detection | ~350ms | ~100ms | 71% faster |
| Lurker Detection | ~400ms | ~100ms | 75% faster |
| Channel Diversity | ~250ms | ~70ms | 72% faster |
| Multi-Client Logins | ~200ms | ~80ms | 60% faster |
| Role Drift | ~180ms | ~90ms | 50% faster |
| Behavior Shifts | ~500ms | ~120ms | 76% faster |
| Reaction Stats | ~220ms | ~85ms | 61% faster |
*Benchmarks on dataset with ~10k events per table, PostgreSQL 14*
_Benchmarks on dataset with ~10k events per table, PostgreSQL 14_
## Query Analysis Tools
@@ -347,6 +368,7 @@ EXPLAIN ANALYZE SELECT ...
```
Key metrics to look for:
- **Seq Scan**: Should be avoided on large tables
- **Index Scan**: Good, indicates proper index usage
- **Execution Time**: Should be < 100ms
@@ -356,25 +378,25 @@ Key metrics to look for:
Admin endpoints for query monitoring:
1. `GET /api/admin/monitoring/database/health`
- Database connection status
- PostgreSQL version
- Database connection status
- PostgreSQL version
2. `GET /api/admin/monitoring/database/tables`
- Table sizes and row counts
- Table sizes and row counts
3. `GET /api/admin/monitoring/database/indexes`
- Index usage statistics
- Unused indexes
- Index usage statistics
- Unused indexes
4. `GET /api/admin/monitoring/database/slow-queries`
- Application-tracked slow queries
- Pagination support
- Application-tracked slow queries
- Pagination support
5. `GET /api/admin/monitoring/database/pg-slow-queries`
- PostgreSQL pg_stat_statements queries
- PostgreSQL pg_stat_statements queries
6. `POST /api/admin/monitoring/database/analyze`
- Run ANALYZE on all tables
- Run ANALYZE on all tables
## Legacy Function Preservation
@@ -391,6 +413,7 @@ export async function getGhostScoresLegacy(guildId: string, since?: Date) {
```
**Benefits:**
- A/B testing capabilities
- Gradual rollout options
- Performance comparison
@@ -399,17 +422,21 @@ export async function getGhostScoresLegacy(guildId: string, since?: Date) {
## Testing Strategy
### Unit Tests
- Pagination utilities: 21 tests passing
- Channel analytics: 8 tests passing
- Mock database responses for optimized queries
### Integration Tests
- Analytics routes: 5 tests passing
- End-to-end query execution
- Proper middleware integration
### Performance Tests
Recommended tests to add:
- Load testing with concurrent requests
- Query performance benchmarks
- Cache hit rate monitoring
@@ -417,73 +444,76 @@ Recommended tests to add:
## Best Practices Applied
1. **Database-level Aggregation**
- Use SQL `GROUP BY`, `COUNT()`, `AVG()`, `SUM()`
- Avoid fetching all rows for in-memory processing
- Use SQL `GROUP BY`, `COUNT()`, `AVG()`, `SUM()`
- Avoid fetching all rows for in-memory processing
2. **Query Limits**
- All queries have `LIMIT` clauses
- Prevents unbounded result sets
- All queries have `LIMIT` clauses
- Prevents unbounded result sets
3. **Index Utilization**
- Queries designed to use existing indexes
- Partial indexes for filtered queries
- Queries designed to use existing indexes
- Partial indexes for filtered queries
4. **N+1 Query Elimination**
- Replaced multiple queries with single queries
- Used JOINs and CTEs instead of separate fetches
- Replaced multiple queries with single queries
- Used JOINs and CTEs instead of separate fetches
5. **Result Set Reduction**
- Only select needed columns
- Filter at database level, not application level
- Only select needed columns
- Filter at database level, not application level
6. **Caching**
- Redis caching for expensive queries
- Appropriate TTLs based on data volatility
- Redis caching for expensive queries
- Appropriate TTLs based on data volatility
7. **Monitoring**
- Slow query logging
- Query performance tracking
- Admin monitoring endpoints
- Slow query logging
- Query performance tracking
- Admin monitoring endpoints
## Future Optimizations
Potential improvements:
1. **Materialized Views**
- Pre-compute complex analytics
- Refresh on schedule or trigger
- Pre-compute complex analytics
- Refresh on schedule or trigger
2. **Table Partitioning**
- Partition large event tables by date
- Improve query performance on time-ranges
- Partition large event tables by date
- Improve query performance on time-ranges
3. **Read Replicas**
- Separate read workload from writes
- Scale read capacity horizontally
- Separate read workload from writes
- Scale read capacity horizontally
4. **Connection Pooling**
- External pooler like PgBouncer
- Better connection management
- External pooler like PgBouncer
- Better connection management
5. **Query Result Caching**
- Cache query results at database level
- Reduce repeated query execution
- Cache query results at database level
- Reduce repeated query execution
## Maintenance
### Regular Tasks
**Weekly:**
- Review slow query logs
- Check cache hit rates
- Monitor query performance trends
**Monthly:**
- Run `ANALYZE` on all tables
- Review index usage statistics
- Check for unused indexes
**Quarterly:**
- Performance benchmarking
- Review and adjust cache TTLs
- Evaluate new optimization opportunities

412
RATE_LIMITING.md
View File

@@ -17,26 +17,28 @@ All API responses include the following rate limit headers:
### Rate Limit Policies
| Endpoint Category | Limit | Window | Description |
|------------------|-------|--------|-------------|
| Global | 100 requests | 15 minutes | Applied to all API endpoints |
| Authentication | 5 requests | 15 minutes | Login and authentication endpoints |
| Analytics | 30 requests | 1 minute | Analytics data endpoints |
| Admin | 100 requests | 15 minutes | Admin-only endpoints |
| Public | 60 requests | 1 minute | Public data endpoints |
| Webhooks | 1000 requests | 1 hour | Webhook endpoints |
| Refresh Token | 10 requests | 15 minutes | Token refresh endpoint |
| Endpoint Category | Limit | Window | Description |
| ----------------- | ------------- | ---------- | ---------------------------------- |
| Global | 100 requests | 15 minutes | Applied to all API endpoints |
| Authentication | 5 requests | 15 minutes | Login and authentication endpoints |
| Analytics | 30 requests | 1 minute | Analytics data endpoints |
| Admin | 100 requests | 15 minutes | Admin-only endpoints |
| Public | 60 requests | 1 minute | Public data endpoints |
| Webhooks | 1000 requests | 1 hour | Webhook endpoints |
| Refresh Token | 10 requests | 15 minutes | Token refresh endpoint |
### User-Based Rate Limiting
Authenticated users have different rate limits based on their subscription tier and role:
**By Subscription Tier:**
- **FREE**: 30 requests/minute, 100 requests/15 minutes
- **PRO**: 100 requests/minute, 1,000 requests/15 minutes
- **ENTERPRISE**: 300 requests/minute, 5,000 requests/15 minutes
**By Role (overrides tier limits):**
- **Admin**: 200 requests/minute
- **Moderator**: 100 requests/minute
- **Unauthenticated**: 30 requests/minute
@@ -47,9 +49,9 @@ When rate limited, the API returns a `429 Too Many Requests` response:
```json
{
"error": "Too many requests",
"message": "Too many requests. Please try again later.",
"retryAfter": 600
"error": "Too many requests",
"message": "Too many requests. Please try again later.",
"retryAfter": 600
}
```
@@ -72,32 +74,32 @@ All authenticated API responses include quota-related headers:
#### FREE Tier
| Category | Daily Limit |
|----------|-------------|
| Analytics | 100 requests |
| API | 1,000 requests |
| Public | 500 requests |
| Admin | No access |
| Category | Daily Limit |
| --------- | ------------------ |
| Analytics | 100 requests |
| API | 1,000 requests |
| Public | 500 requests |
| Admin | No access |
| **Total** | **1,000 requests** |
#### PRO Tier
| Category | Daily Limit |
|----------|-------------|
| Analytics | 1,000 requests |
| API | 10,000 requests |
| Public | 5,000 requests |
| Admin | No access |
| Category | Daily Limit |
| --------- | ------------------- |
| Analytics | 1,000 requests |
| API | 10,000 requests |
| Public | 5,000 requests |
| Admin | No access |
| **Total** | **10,000 requests** |
#### ENTERPRISE Tier
| Category | Daily Limit |
|----------|-------------|
| Analytics | 10,000 requests |
| API | 100,000 requests |
| Public | 50,000 requests |
| Admin | 50,000 requests |
| Category | Daily Limit |
| --------- | -------------------- |
| Analytics | 10,000 requests |
| API | 100,000 requests |
| Public | 50,000 requests |
| Admin | 50,000 requests |
| **Total** | **100,000 requests** |
### Endpoint Categories
@@ -115,14 +117,14 @@ When quota is exceeded, the API returns a `429 Too Many Requests` response:
```json
{
"error": "Quota exceeded",
"message": "You have exceeded your analytics quota for the day. Please upgrade your subscription or try again tomorrow.",
"quota": {
"limit": 100,
"remaining": 0,
"reset": 43200,
"category": "analytics"
}
"error": "Quota exceeded",
"message": "You have exceeded your analytics quota for the day. Please upgrade your subscription or try again tomorrow.",
"quota": {
"limit": 100,
"remaining": 0,
"reset": 43200,
"category": "analytics"
}
}
```
@@ -137,35 +139,36 @@ View your current quota usage across all categories.
**Authentication**: Required (JWT or API key)
**Response**:
```json
{
"tier": "FREE",
"usage": {
"analytics": {
"used": 50,
"limit": 100,
"remaining": 50
"tier": "FREE",
"usage": {
"analytics": {
"used": 50,
"limit": 100,
"remaining": 50
},
"api": {
"used": 200,
"limit": 1000,
"remaining": 800
},
"total": {
"used": 250,
"limit": 1000,
"remaining": 750
}
},
"api": {
"used": 200,
"limit": 1000,
"remaining": 800
"limits": {
"analytics": { "requests": 100, "window": "daily" },
"api": { "requests": 1000, "window": "daily" },
"total": { "requests": 1000, "window": "daily" }
},
"total": {
"used": 250,
"limit": 1000,
"remaining": 750
"rateLimits": {
"requestsPerMinute": 30,
"requestsPer15Minutes": 100
}
},
"limits": {
"analytics": { "requests": 100, "window": "daily" },
"api": { "requests": 1000, "window": "daily" },
"total": { "requests": 1000, "window": "daily" }
},
"rateLimits": {
"requestsPerMinute": 30,
"requestsPer15Minutes": 100
}
}
```
@@ -178,6 +181,7 @@ View quota and rate limits for all subscription tiers.
**Authentication**: None required
**Response**:
```json
{
"FREE": {
@@ -211,6 +215,7 @@ View quota usage for a specific user.
**Authentication**: Required (Admin role)
**Response**:
```json
{
"user": {
@@ -244,21 +249,23 @@ Change a user's subscription tier.
**Authentication**: Required (Admin role)
**Request Body**:
```json
{
"tier": "PRO"
"tier": "PRO"
}
```
**Response**:
```json
{
"message": "User tier updated successfully",
"user": {
"id": "user123",
"username": "johndoe",
"subscriptionTier": "PRO"
}
"message": "User tier updated successfully",
"user": {
"id": "user123",
"username": "johndoe",
"subscriptionTier": "PRO"
}
}
```
@@ -271,15 +278,17 @@ Reset quota usage for a user. Optionally specify a category to reset only that c
**Authentication**: Required (Admin role)
**Query Parameters**:
- `category` (optional): Specific category to reset (analytics, api, admin, public, total)
**Response**:
```json
{
"message": "Quota reset for category: analytics",
"userId": "user123",
"username": "johndoe",
"category": "analytics"
"message": "Quota reset for category: analytics",
"userId": "user123",
"username": "johndoe",
"category": "analytics"
}
```
@@ -295,6 +304,7 @@ Reset quota usage for a user. Optionally specify a category to reset only that c
### Prerequisites
All IP management endpoints require:
- Authentication (valid JWT token)
- Admin role
@@ -307,15 +317,16 @@ Get all permanently blocked IP addresses.
**Endpoint**: `GET /blocked`
**Response**:
```json
{
"blocked": [
{
"ip": "192.168.1.1",
"reason": "Malicious activity",
"createdAt": "2024-01-01T00:00:00.000Z"
}
]
"blocked": [
{
"ip": "192.168.1.1",
"reason": "Malicious activity",
"createdAt": "2024-01-01T00:00:00.000Z"
}
]
}
```
@@ -326,15 +337,16 @@ Get all whitelisted IP addresses.
**Endpoint**: `GET /whitelisted`
**Response**:
```json
{
"whitelisted": [
{
"ip": "192.168.1.100",
"reason": "Office IP",
"createdAt": "2024-01-01T00:00:00.000Z"
}
]
"whitelisted": [
{
"ip": "192.168.1.100",
"reason": "Office IP",
"createdAt": "2024-01-01T00:00:00.000Z"
}
]
}
```
@@ -345,20 +357,23 @@ Check the blocking/whitelisting status of a specific IP.
**Endpoint**: `GET /check/:ip`
**Parameters**:
- `ip` (path): IP address to check (IPv4 or IPv6)
**Response**:
```json
{
"ip": "192.168.1.1",
"blocked": false,
"whitelisted": false,
"violations": 5,
"status": "normal"
"ip": "192.168.1.1",
"blocked": false,
"whitelisted": false,
"violations": 5,
"status": "normal"
}
```
**Status Values**:
- `normal`: IP is neither blocked nor whitelisted
- `blocked`: IP is permanently blocked
- `whitelisted`: IP is whitelisted
@@ -370,19 +385,21 @@ Permanently block an IP address.
**Endpoint**: `POST /block`
**Request Body**:
```json
{
"ip": "192.168.1.1",
"reason": "Malicious activity detected"
"ip": "192.168.1.1",
"reason": "Malicious activity detected"
}
```
**Response**:
```json
{
"message": "IP permanently blocked",
"ip": "192.168.1.1",
"reason": "Malicious activity detected"
"message": "IP permanently blocked",
"ip": "192.168.1.1",
"reason": "Malicious activity detected"
}
```
@@ -393,26 +410,29 @@ Temporarily block an IP address for a specified duration.
**Endpoint**: `POST /temp-block`
**Request Body**:
```json
{
"ip": "192.168.1.1",
"duration": 3600,
"reason": "Rate limit abuse"
"ip": "192.168.1.1",
"duration": 3600,
"reason": "Rate limit abuse"
}
```
**Parameters**:
- `ip`: IP address to block
- `duration`: Block duration in seconds (60-86400)
- `reason`: (optional) Reason for blocking
**Response**:
```json
{
"message": "IP temporarily blocked",
"ip": "192.168.1.1",
"duration": 3600,
"reason": "Rate limit abuse"
"message": "IP temporarily blocked",
"ip": "192.168.1.1",
"duration": 3600,
"reason": "Rate limit abuse"
}
```
@@ -423,13 +443,15 @@ Remove a permanent IP block.
**Endpoint**: `DELETE /unblock/:ip`
**Parameters**:
- `ip` (path): IP address to unblock
**Response**:
```json
{
"message": "IP unblocked successfully",
"ip": "192.168.1.1"
"message": "IP unblocked successfully",
"ip": "192.168.1.1"
}
```
@@ -440,13 +462,15 @@ Remove a temporary IP block.
**Endpoint**: `DELETE /temp-unblock/:ip`
**Parameters**:
- `ip` (path): IP address to unblock
**Response**:
```json
{
"message": "Temporary block removed successfully",
"ip": "192.168.1.1"
"message": "Temporary block removed successfully",
"ip": "192.168.1.1"
}
```
@@ -457,19 +481,21 @@ Add an IP to the whitelist, bypassing all rate limits and blocks.
**Endpoint**: `POST /whitelist`
**Request Body**:
```json
{
"ip": "192.168.1.100",
"reason": "Office IP address"
"ip": "192.168.1.100",
"reason": "Office IP address"
}
```
**Response**:
```json
{
"message": "IP added to whitelist",
"ip": "192.168.1.100",
"reason": "Office IP address"
"message": "IP added to whitelist",
"ip": "192.168.1.100",
"reason": "Office IP address"
}
```
@@ -480,13 +506,15 @@ Remove an IP from the whitelist.
**Endpoint**: `DELETE /whitelist/:ip`
**Parameters**:
- `ip` (path): IP address to remove from whitelist
**Response**:
```json
{
"message": "IP removed from whitelist",
"ip": "192.168.1.100"
"message": "IP removed from whitelist",
"ip": "192.168.1.100"
}
```
@@ -495,6 +523,7 @@ Remove an IP from the whitelist.
### Prerequisites
All monitoring endpoints require:
- Authentication (valid JWT token)
- Admin role
@@ -507,32 +536,33 @@ Get overall rate limit statistics and violations.
**Endpoint**: `GET /rate-limits`
**Response**:
```json
{
"violations": [
{
"ip": "192.168.1.1",
"count": 15,
"ttl": 3456
"violations": [
{
"ip": "192.168.1.1",
"count": 15,
"ttl": 3456
}
],
"tempBlocked": [
{
"ip": "192.168.1.2",
"ttl": 1800
}
],
"rateLimitStats": {
"global": 45,
"auth": 12,
"analytics": 8
},
"summary": {
"totalViolations": 27,
"uniqueIPsWithViolations": 5,
"tempBlockedCount": 2,
"activeRateLimiters": 3
}
],
"tempBlocked": [
{
"ip": "192.168.1.2",
"ttl": 1800
}
],
"rateLimitStats": {
"global": 45,
"auth": 12,
"analytics": 8
},
"summary": {
"totalViolations": 27,
"uniqueIPsWithViolations": 5,
"tempBlockedCount": 2,
"activeRateLimiters": 3
}
}
```
@@ -543,22 +573,24 @@ Get detailed rate limit information for a specific IP.
**Endpoint**: `GET /rate-limits/:ip`
**Parameters**:
- `ip` (path): IP address to check
**Response**:
```json
{
"ip": "192.168.1.1",
"violations": 15,
"violationTTL": 3456,
"isTemporarilyBlocked": false,
"blockTTL": null,
"rateLimitInfo": {
"rl:global:192.168.1.1": {
"value": "45",
"ttl": 854
"ip": "192.168.1.1",
"violations": 15,
"violationTTL": 3456,
"isTemporarilyBlocked": false,
"blockTTL": null,
"rateLimitInfo": {
"rl:global:192.168.1.1": {
"value": "45",
"ttl": 854
}
}
}
}
```
@@ -569,14 +601,16 @@ Clear rate limit violations and data for a specific IP.
**Endpoint**: `DELETE /rate-limits/:ip`
**Parameters**:
- `ip` (path): IP address to clear data for
**Response**:
```json
{
"message": "Rate limit data cleared successfully",
"ip": "192.168.1.1",
"clearedKeys": 3
"message": "Rate limit data cleared successfully",
"ip": "192.168.1.1",
"clearedKeys": 3
}
```
@@ -587,35 +621,36 @@ Get system health and performance metrics.
**Endpoint**: `GET /system`
**Response**:
```json
{
"status": "healthy",
"timestamp": "2024-01-01T00:00:00.000Z",
"uptime": "86400s",
"system": {
"cpu": {
"usage": "45.2%",
"cores": 4,
"load": [0.8, 0.7, 0.6]
"status": "healthy",
"timestamp": "2024-01-01T00:00:00.000Z",
"uptime": "86400s",
"system": {
"cpu": {
"usage": "45.2%",
"cores": 4,
"load": [0.8, 0.7, 0.6]
},
"memory": {
"usage": "62.3%",
"free": "2048MB",
"total": "8192MB"
},
"process": {
"memory": {
"rss": 125829120,
"heapTotal": 67584000,
"heapUsed": 45678912,
"external": 1234567
},
"pid": 12345
}
},
"memory": {
"usage": "62.3%",
"free": "2048MB",
"total": "8192MB"
},
"process": {
"memory": {
"rss": 125829120,
"heapTotal": 67584000,
"heapUsed": 45678912,
"external": 1234567
},
"pid": 12345
"redis": {
"available": true
}
},
"redis": {
"available": true
}
}
```
@@ -666,7 +701,7 @@ Invalid request format or parameters.
```json
{
"error": "Invalid IP address format"
"error": "Invalid IP address format"
}
```
@@ -676,8 +711,8 @@ IP address is blocked.
```json
{
"error": "Access denied from this IP",
"reason": "IP address permanently blocked"
"error": "Access denied from this IP",
"reason": "IP address permanently blocked"
}
```
@@ -687,9 +722,9 @@ Request exceeds size limit.
```json
{
"error": "Request entity too large",
"maxSize": "10MB",
"received": "15.2MB"
"error": "Request entity too large",
"maxSize": "10MB",
"received": "15.2MB"
}
```
@@ -699,9 +734,9 @@ Rate limit exceeded.
```json
{
"error": "Too many requests",
"message": "Too many authentication attempts. Please try again later.",
"retryAfter": 600
"error": "Too many requests",
"message": "Too many authentication attempts. Please try again later.",
"retryAfter": 600
}
```
@@ -711,9 +746,9 @@ Service temporarily unavailable due to high load or maintenance.
```json
{
"error": "Service temporarily unavailable",
"message": "Server under high load, please try again later",
"retryAfter": 60
"error": "Service temporarily unavailable",
"message": "Server under high load, please try again later",
"retryAfter": 60
}
```
@@ -793,6 +828,7 @@ model WhitelistedIP {
## Support
For issues or questions about rate limiting and DDoS protection, please refer to:
- [SECURITY.md](../SECURITY.md) - Security policies
- [GitHub Issues](https://github.com/subculture-collective/discord-spywatcher/issues) - Report issues
- [Contributing Guide](../CONTRIBUTING.md) - Contribution guidelines

124
README.md
View File

@@ -54,6 +54,7 @@ Explore and test the API using our interactive documentation portals:
- **[OpenAPI Spec](http://localhost:3001/api/openapi.json)** - Raw OpenAPI 3.0 specification
**Quick Links:**
- 📖 [Interactive API Guide](./docs/api/INTERACTIVE_API_GUIDE.md) - Code examples in 6+ languages
- 🔐 [Authentication Guide](./docs/api/AUTHENTICATION_GUIDE.md) - OAuth2 setup and JWT management
- ⚡ [Rate Limiting Guide](./docs/api/RATE_LIMITING_GUIDE.md) - Best practices and optimization
@@ -72,6 +73,7 @@ npm run docs:preview
```
Documentation is built with [VitePress](https://vitepress.dev/) and includes:
- Interactive search
- Dark mode support
- Mobile-responsive design
@@ -106,6 +108,7 @@ docker-compose -f docker-compose.dev.yml up
```
Access the application:
- Frontend: http://localhost:5173
- Backend API: http://localhost:3001
- PostgreSQL: localhost:5432
@@ -175,37 +178,38 @@ Copy `backend/.env.example` to `backend/.env` and configure the following variab
#### Required Variables
| Variable | Description | Example | Validation |
|----------|-------------|---------|------------|
| `DISCORD_BOT_TOKEN` | Discord bot token from Developer Portal | `MTk...` | Min 50 characters |
| `DISCORD_CLIENT_ID` | OAuth2 client ID | `123456789` | Min 10 characters |
| `DISCORD_CLIENT_SECRET` | OAuth2 client secret | `abc123...` | Min 20 characters |
| `DISCORD_REDIRECT_URI` | OAuth2 redirect URI | `http://localhost:5173/auth/callback` | Valid URL |
| `JWT_SECRET` | Secret for signing access tokens | `random-32-char-string` | Min 32 characters |
| `JWT_REFRESH_SECRET` | Secret for signing refresh tokens | `another-32-char-string` | Min 32 characters |
| Variable | Description | Example | Validation |
| ----------------------- | --------------------------------------- | ------------------------------------- | ----------------- |
| `DISCORD_BOT_TOKEN` | Discord bot token from Developer Portal | `MTk...` | Min 50 characters |
| `DISCORD_CLIENT_ID` | OAuth2 client ID | `123456789` | Min 10 characters |
| `DISCORD_CLIENT_SECRET` | OAuth2 client secret | `abc123...` | Min 20 characters |
| `DISCORD_REDIRECT_URI` | OAuth2 redirect URI | `http://localhost:5173/auth/callback` | Valid URL |
| `JWT_SECRET` | Secret for signing access tokens | `random-32-char-string` | Min 32 characters |
| `JWT_REFRESH_SECRET` | Secret for signing refresh tokens | `another-32-char-string` | Min 32 characters |
#### Optional Variables
| Variable | Description | Default | Validation |
|----------|-------------|---------|------------|
| `NODE_ENV` | Environment mode | `development` | `development`, `staging`, `production`, `test` |
| `PORT` | Server port | `3001` | Positive integer |
| `DATABASE_URL` | PostgreSQL connection string | - | Valid URL |
| `DISCORD_GUILD_ID` | Optional specific guild ID | - | String |
| `BOT_GUILD_IDS` | Comma-separated guild IDs to monitor | - | Comma-separated list |
| `ADMIN_DISCORD_IDS` | Comma-separated admin user IDs | - | Comma-separated list |
| `CORS_ORIGINS` | Comma-separated allowed origins | `http://localhost:5173` | Comma-separated URLs |
| `JWT_ACCESS_EXPIRES_IN` | Access token expiration | `15m` | Time string |
| `JWT_REFRESH_EXPIRES_IN` | Refresh token expiration | `7d` | Time string |
| `REDIS_URL` | Redis connection string | `redis://localhost:6379` | Valid URL |
| `ENABLE_RATE_LIMITING` | Enable rate limiting | `true` | `true` or `false` |
| `ENABLE_REDIS_RATE_LIMITING` | Enable Redis-backed rate limiting | `true` | `true` or `false` |
| `ENABLE_IP_BLOCKING` | Enable IP blocking | `true` | `true` or `false` |
| `ENABLE_LOAD_SHEDDING` | Enable load shedding under high load | `true` | `true` or `false` |
| `LOG_LEVEL` | Logging level | `info` | `error`, `warn`, `info`, `debug` |
| `FRONTEND_URL` | Frontend URL for redirects | - | Valid URL |
| Variable | Description | Default | Validation |
| ---------------------------- | ------------------------------------ | ------------------------ | ---------------------------------------------- |
| `NODE_ENV` | Environment mode | `development` | `development`, `staging`, `production`, `test` |
| `PORT` | Server port | `3001` | Positive integer |
| `DATABASE_URL` | PostgreSQL connection string | - | Valid URL |
| `DISCORD_GUILD_ID` | Optional specific guild ID | - | String |
| `BOT_GUILD_IDS` | Comma-separated guild IDs to monitor | - | Comma-separated list |
| `ADMIN_DISCORD_IDS` | Comma-separated admin user IDs | - | Comma-separated list |
| `CORS_ORIGINS` | Comma-separated allowed origins | `http://localhost:5173` | Comma-separated URLs |
| `JWT_ACCESS_EXPIRES_IN` | Access token expiration | `15m` | Time string |
| `JWT_REFRESH_EXPIRES_IN` | Refresh token expiration | `7d` | Time string |
| `REDIS_URL` | Redis connection string | `redis://localhost:6379` | Valid URL |
| `ENABLE_RATE_LIMITING` | Enable rate limiting | `true` | `true` or `false` |
| `ENABLE_REDIS_RATE_LIMITING` | Enable Redis-backed rate limiting | `true` | `true` or `false` |
| `ENABLE_IP_BLOCKING` | Enable IP blocking | `true` | `true` or `false` |
| `ENABLE_LOAD_SHEDDING` | Enable load shedding under high load | `true` | `true` or `false` |
| `LOG_LEVEL` | Logging level | `info` | `error`, `warn`, `info`, `debug` |
| `FRONTEND_URL` | Frontend URL for redirects | - | Valid URL |
**Generate secure secrets:**
```bash
# Generate JWT secrets
node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
@@ -215,15 +219,16 @@ node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
Copy `frontend/.env.example` to `frontend/.env` and configure:
| Variable | Description | Example | Required |
|----------|-------------|---------|----------|
| `VITE_API_URL` | Backend API base URL | `http://localhost:3001/api` | Yes |
| `VITE_DISCORD_CLIENT_ID` | Discord OAuth2 client ID | `123456789` | Yes |
| `VITE_ENVIRONMENT` | Environment mode | `development` | No (default: `development`) |
| `VITE_ENABLE_ANALYTICS` | Enable analytics tracking | `false` | No (default: `false`) |
| `VITE_ANALYTICS_TRACKING_ID` | Analytics tracking ID | - | No |
| Variable | Description | Example | Required |
| ---------------------------- | ------------------------- | --------------------------- | --------------------------- |
| `VITE_API_URL` | Backend API base URL | `http://localhost:3001/api` | Yes |
| `VITE_DISCORD_CLIENT_ID` | Discord OAuth2 client ID | `123456789` | Yes |
| `VITE_ENVIRONMENT` | Environment mode | `development` | No (default: `development`) |
| `VITE_ENABLE_ANALYTICS` | Enable analytics tracking | `false` | No (default: `false`) |
| `VITE_ANALYTICS_TRACKING_ID` | Analytics tracking ID | - | No |
**Important Notes:**
- All frontend variables must be prefixed with `VITE_` to be exposed to the browser
- Never include secrets in frontend environment variables
- Frontend variables are embedded in the build and publicly accessible
@@ -232,12 +237,14 @@ Copy `frontend/.env.example` to `frontend/.env` and configure:
### Environment Validation
The backend uses [Zod](https://zod.dev/) for runtime environment validation:
- All required variables are validated on startup
- Type safety is enforced (strings, numbers, URLs, enums)
- Clear error messages for missing or invalid configuration
- Application exits with code 1 if validation fails
Example validation error:
```
❌ Invalid environment configuration:
@@ -258,6 +265,7 @@ Spywatcher implements comprehensive security measures to protect against abuse a
- **Load management** with circuit breakers and load shedding under high load
See [RATE_LIMITING.md](./RATE_LIMITING.md) for detailed documentation on:
- Rate limiting configuration
- Endpoint-specific limits
- DDoS protection mechanisms
@@ -274,6 +282,7 @@ Spywatcher uses PostgreSQL with PgBouncer for efficient connection pooling and r
- **Graceful shutdown** - Proper connection cleanup on application shutdown
See [CONNECTION_POOLING.md](./CONNECTION_POOLING.md) for detailed documentation on:
- PgBouncer setup and configuration
- Connection pool monitoring and metrics
- Database health checks and alerting
@@ -281,6 +290,7 @@ See [CONNECTION_POOLING.md](./CONNECTION_POOLING.md) for detailed documentation
- Troubleshooting connection issues
Additional database documentation:
- [POSTGRESQL.md](./POSTGRESQL.md) - PostgreSQL setup and management
- [DATABASE_OPTIMIZATION.md](./DATABASE_OPTIMIZATION.md) - Query optimization and indexing
- [docs/PGBOUNCER_SETUP.md](./docs/PGBOUNCER_SETUP.md) - Quick reference guide
@@ -297,10 +307,12 @@ Spywatcher implements comprehensive backup and disaster recovery procedures to e
- **Recovery Procedures** - Documented runbooks for various disaster scenarios
**Recovery Objectives:**
- **RTO** (Recovery Time Objective): < 4 hours
- **RPO** (Recovery Point Objective): < 1 hour
See [DISASTER_RECOVERY.md](./DISASTER_RECOVERY.md) for detailed documentation on:
- Backup strategy and configuration
- Automated backup scripts and schedules
- WAL archiving setup for PITR
@@ -309,6 +321,7 @@ See [DISASTER_RECOVERY.md](./DISASTER_RECOVERY.md) for detailed documentation on
- Monitoring and alerting setup
**Quick Commands:**
```bash
# Manual database backup
cd backend && npm run db:backup
@@ -336,6 +349,7 @@ Spywatcher includes comprehensive monitoring and observability features:
- **Grafana** - Unified dashboards for logs and metrics
See [MONITORING.md](./MONITORING.md) for detailed documentation on:
- Sentry configuration and error tracking
- Prometheus metrics and custom instrumentation
- Health check endpoints
@@ -344,6 +358,7 @@ See [MONITORING.md](./MONITORING.md) for detailed documentation on:
- Grafana dashboard creation
See [LOGGING.md](./LOGGING.md) for centralized logging documentation:
- Log aggregation with Grafana Loki
- Log search and filtering with LogQL
- Log retention policies (30-day default)
@@ -400,8 +415,8 @@ npm install @spywatcher/sdk
import { Spywatcher } from '@spywatcher/sdk';
const client = new Spywatcher({
baseUrl: 'https://api.spywatcher.com/api',
apiKey: 'spy_live_your_api_key_here'
baseUrl: 'https://api.spywatcher.com/api',
apiKey: 'spy_live_your_api_key_here',
});
// Get ghost users
@@ -414,9 +429,9 @@ const suspicions = await client.getSuspicionData();
### API Documentation
- **[Interactive API Documentation](./docs/API_DOCUMENTATION.md)** - OpenAPI/Swagger docs with screenshots
- **Swagger UI**: `/api/docs` - Interactive testing interface
- **ReDoc**: `/api/redoc` - Clean, professional documentation view
- **OpenAPI Spec**: `/api/openapi.json` - Raw OpenAPI 3.0 specification
- **Swagger UI**: `/api/docs` - Interactive testing interface
- **ReDoc**: `/api/redoc` - Clean, professional documentation view
- **OpenAPI Spec**: `/api/openapi.json` - Raw OpenAPI 3.0 specification
- **[Public API Reference](./docs/PUBLIC_API.md)** - Complete API documentation with examples
- **[Developer Guide](./docs/DEVELOPER_GUIDE.md)** - Step-by-step guide for building integrations
- **[SDK Documentation](./sdk/README.md)** - TypeScript/JavaScript SDK usage guide
@@ -473,16 +488,16 @@ npm run dev:api
The repository includes three example plugins:
1. **Message Logger** (`backend/plugins/examples/message-logger/`)
- Logs all Discord messages to a file
- Demonstrates basic plugin structure and Discord event hooks
- Logs all Discord messages to a file
- Demonstrates basic plugin structure and Discord event hooks
2. **Analytics Extension** (`backend/plugins/examples/analytics-extension/`)
- Adds custom analytics API endpoints
- Shows database access and Redis caching
- Adds custom analytics API endpoints
- Shows database access and Redis caching
3. **Webhook Notifier** (`backend/plugins/examples/webhook-notifier/`)
- Sends notifications to external webhooks
- Demonstrates network access and event monitoring
- Sends notifications to external webhooks
- Demonstrates network access and event monitoring
### Plugin Management API
@@ -583,15 +598,15 @@ Spywatcher includes comprehensive production deployment infrastructure with Kube
### Infrastructure as Code
- **Terraform**: Complete AWS infrastructure modules
- VPC with multi-AZ setup
- EKS Kubernetes cluster
- RDS PostgreSQL (Multi-AZ, encrypted)
- ElastiCache Redis (encrypted, failover)
- Application Load Balancer with WAF
- VPC with multi-AZ setup
- EKS Kubernetes cluster
- RDS PostgreSQL (Multi-AZ, encrypted)
- ElastiCache Redis (encrypted, failover)
- Application Load Balancer with WAF
- **Kubernetes**: Production-ready manifests
- Auto-scaling with HorizontalPodAutoscaler
- Health checks and pod disruption budgets
- Security contexts and network policies
- Auto-scaling with HorizontalPodAutoscaler
- Health checks and pod disruption budgets
- Security contexts and network policies
- **Helm Charts**: Simplified deployment and configuration
### Quick Deployment
@@ -625,6 +640,7 @@ helm install spywatcher ./helm/spywatcher -n spywatcher
### CI/CD Pipeline
GitHub Actions workflows for automated deployment:
- Docker image building and pushing to GHCR
- Database migrations
- Multiple deployment strategy support
@@ -635,7 +651,9 @@ See [.github/workflows/deploy-production.yml](./.github/workflows/deploy-product
## 👥 Contributions
See [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines on contributing to this project.
We welcome contributions from everyone! Please read our [Contributing Guidelines](./CONTRIBUTING.md) to get started.
This project follows a [Code of Conduct](./CODE_OF_CONDUCT.md) to ensure a welcoming environment for all contributors.
---

125
REDIS_CACHING.md
View File

@@ -25,15 +25,20 @@ The `CacheService` provides a high-level API for caching with the following feat
const value = await cache.get<T>(key);
// Set value with TTL and tags
await cache.set(key, value, {
ttl: 300, // 5 minutes
tags: ['guild:123', 'analytics:ghosts']
await cache.set(key, value, {
ttl: 300, // 5 minutes
tags: ['guild:123', 'analytics:ghosts'],
});
// Remember pattern - cache or execute callback
const result = await cache.remember(key, 300, async () => {
return await expensiveOperation();
}, { tags: ['my-tag'] });
const result = await cache.remember(
key,
300,
async () => {
return await expensiveOperation();
},
{ tags: ['my-tag'] }
);
// Invalidate by tag
await cache.invalidateByTag('guild:123');
@@ -52,7 +57,7 @@ await pubsub.publish('channel-name', { data: 'value' });
// Subscribe to channel
await pubsub.subscribe('channel-name', (message) => {
console.log('Received:', message);
console.log('Received:', message);
});
// Specialized helpers
@@ -78,16 +83,17 @@ await cacheInvalidation.onRoleChanged(guildId);
All analytics endpoints use the cache-aside pattern with appropriate TTLs:
| Endpoint | Function | TTL | Cache Key Pattern | Tags |
|----------|----------|-----|-------------------|------|
| Ghost Scores | `getGhostScores` | 5 min | `analytics:ghosts:{guildId}:{since}` | `guild:{guildId}`, `analytics:ghosts` |
| Lurker Flags | `getLurkerFlags` | 5 min | `analytics:lurkers:{guildId}:{since}` | `guild:{guildId}`, `analytics:lurkers` |
| Heatmap | `getChannelHeatmap` | 15 min | `analytics:heatmap:{guildId}:{since}` | `guild:{guildId}`, `analytics:heatmap` |
| Role Drift | `getRoleDriftFlags` | 10 min | `analytics:roles:{guildId}:{since}` | `guild:{guildId}`, `analytics:roles` |
| Client Drift | `getClientDriftFlags` | 2 min | `analytics:clients:{guildId}:{since}` | `guild:{guildId}`, `analytics:clients` |
| Behavior Shifts | `getBehaviorShiftFlags` | 5 min | `analytics:shifts:{guildId}:{since}` | `guild:{guildId}`, `analytics:shifts` |
| Endpoint | Function | TTL | Cache Key Pattern | Tags |
| --------------- | ----------------------- | ------ | ------------------------------------- | -------------------------------------- |
| Ghost Scores | `getGhostScores` | 5 min | `analytics:ghosts:{guildId}:{since}` | `guild:{guildId}`, `analytics:ghosts` |
| Lurker Flags | `getLurkerFlags` | 5 min | `analytics:lurkers:{guildId}:{since}` | `guild:{guildId}`, `analytics:lurkers` |
| Heatmap | `getChannelHeatmap` | 15 min | `analytics:heatmap:{guildId}:{since}` | `guild:{guildId}`, `analytics:heatmap` |
| Role Drift | `getRoleDriftFlags` | 10 min | `analytics:roles:{guildId}:{since}` | `guild:{guildId}`, `analytics:roles` |
| Client Drift | `getClientDriftFlags` | 2 min | `analytics:clients:{guildId}:{since}` | `guild:{guildId}`, `analytics:clients` |
| Behavior Shifts | `getBehaviorShiftFlags` | 5 min | `analytics:shifts:{guildId}:{since}` | `guild:{guildId}`, `analytics:shifts` |
**Rationale:**
- **Ghost/Lurker/Shifts**: 5 minutes - Moderate volatility, balance freshness with DB load
- **Heatmap**: 15 minutes - Slower-changing aggregate data
- **Roles**: 10 minutes - Role changes are infrequent
@@ -162,16 +168,19 @@ For high availability in production, use Redis Cluster:
```typescript
// backend/src/utils/redis.ts
const redisCluster = new Redis.Cluster([
{ host: 'redis-node-1', port: 6379 },
{ host: 'redis-node-2', port: 6379 },
{ host: 'redis-node-3', port: 6379 },
], {
redisOptions: {
password: process.env.REDIS_PASSWORD,
tls: process.env.NODE_ENV === 'production' ? {} : undefined,
},
});
const redisCluster = new Redis.Cluster(
[
{ host: 'redis-node-1', port: 6379 },
{ host: 'redis-node-2', port: 6379 },
{ host: 'redis-node-3', port: 6379 },
],
{
redisOptions: {
password: process.env.REDIS_PASSWORD,
tls: process.env.NODE_ENV === 'production' ? {} : undefined,
},
}
);
```
### Memory Management
@@ -191,18 +200,19 @@ maxmemory-policy allkeys-lru
`GET /api/admin/monitoring/cache/stats`
Returns:
```json
{
"stats": {
"hits": 1000,
"misses": 200,
"hitRate": 83.33,
"memoryUsed": "2.5M",
"evictedKeys": 5,
"keyCount": 150
},
"status": "healthy",
"timestamp": "2025-10-29T19:00:00.000Z"
"stats": {
"hits": 1000,
"misses": 200,
"hitRate": 83.33,
"memoryUsed": "2.5M",
"evictedKeys": 5,
"keyCount": 150
},
"status": "healthy",
"timestamp": "2025-10-29T19:00:00.000Z"
}
```
@@ -227,6 +237,7 @@ console.log(`Cache get error for key ${key}:`, error);
### 1. Use Appropriate TTLs
Choose TTLs based on:
- Data volatility
- Query cost
- Freshness requirements
@@ -237,8 +248,8 @@ Always include tags for efficient invalidation:
```typescript
await cache.set(key, value, {
ttl: 300,
tags: [`guild:${guildId}`, `analytics:${type}`]
ttl: 300,
tags: [`guild:${guildId}`, `analytics:${type}`],
});
```
@@ -272,8 +283,8 @@ const data = await cache.remember(key, ttl, () => fetchData());
// Verbose ❌
let data = await cache.get(key);
if (!data) {
data = await fetchData();
await cache.set(key, data, { ttl });
data = await fetchData();
await cache.set(key, data, { ttl });
}
```
@@ -286,8 +297,8 @@ The PubSub service is ready for WebSocket integration:
```typescript
// Subscribe to analytics updates
pubsub.subscribe(`analytics:ghosts:${guildId}`, (data) => {
// Broadcast to WebSocket clients
io.to(guildId).emit('analytics:update', data);
// Broadcast to WebSocket clients
io.to(guildId).emit('analytics:update', data);
});
// Publish updates when cache is invalidated
@@ -299,19 +310,21 @@ await pubsub.publishAnalyticsUpdate(guildId, 'ghosts', freshData);
### Cache Not Working
1. **Check Redis Connection**
```bash
redis-cli ping # Should return PONG
```
```bash
redis-cli ping # Should return PONG
```
2. **Verify Environment Variables**
```bash
echo $REDIS_URL
echo $ENABLE_REDIS_RATE_LIMITING
```
```bash
echo $REDIS_URL
echo $ENABLE_REDIS_RATE_LIMITING
```
3. **Check Logs**
- Look for "Redis connected successfully"
- Look for "Redis connection error"
- Look for "Redis connected successfully"
- Look for "Redis connection error"
### Low Hit Rate
@@ -393,12 +406,12 @@ import { cache } from './services/cache';
// Warm cache with initial data
await cache.warm([
{
key: 'analytics:ghosts:123:all',
value: await fetchGhostData('123'),
options: { ttl: 300, tags: ['guild:123'] }
},
// ... more entries
{
key: 'analytics:ghosts:123:all',
value: await fetchGhostData('123'),
options: { ttl: 300, tags: ['guild:123'] },
},
// ... more entries
]);
```

75
SDK_IMPLEMENTATION.md
View File

@@ -13,6 +13,7 @@ The Public API and SDK provide a comprehensive solution for third-party integrat
A full-featured SDK package with:
#### Features
- **Full TypeScript Support**: Complete type definitions for all API endpoints and data types
- **Promise-based API**: Modern async/await syntax
- **Automatic Error Handling**: Custom error classes for different failure scenarios
@@ -20,11 +21,13 @@ A full-featured SDK package with:
- **Debug Logging**: Optional debug mode for development
#### API Modules
- `AnalyticsAPI`: Ghost users, lurkers, heatmaps, role changes, client data, status shifts
- `Spywatcher` (main class): Timeline, suspicion data, bans, auth & user management
- `SpywatcherClient`: Base HTTP client with error handling
#### Package Details
- **Name**: `@spywatcher/sdk`
- **Version**: 1.0.0
- **Formats**: CommonJS, ES Modules, TypeScript definitions
@@ -36,24 +39,25 @@ A full-featured SDK package with:
New backend routes for API documentation and testing:
- **`/api/public/docs`**: Complete API documentation in JSON format
- Includes all endpoints, parameters, response types
- Code examples in cURL, JavaScript, and Python
- SDK installation instructions
- Rate limit information
- Includes all endpoints, parameters, response types
- Code examples in cURL, JavaScript, and Python
- SDK installation instructions
- Rate limit information
- **`/api/public/openapi`**: OpenAPI 3.0 specification
- Machine-readable API specification
- Compatible with Swagger UI and other OpenAPI tools
- Machine-readable API specification
- Compatible with Swagger UI and other OpenAPI tools
- **`/api/public/test`**: Authentication test endpoint
- Verifies API key is working correctly
- Returns authenticated user information
- Verifies API key is working correctly
- Returns authenticated user information
### 3. Comprehensive Documentation
Three major documentation files:
#### PUBLIC_API.md
- Complete API reference with all endpoints
- Request/response examples
- Error handling guide
@@ -62,6 +66,7 @@ Three major documentation files:
- Code examples in multiple languages
#### DEVELOPER_GUIDE.md
- Step-by-step getting started guide
- Quick start examples
- Common use cases with code
@@ -69,6 +74,7 @@ Three major documentation files:
- Troubleshooting guide
#### SDK README.md
- Installation instructions
- Quick start guide
- Complete API reference
@@ -89,12 +95,14 @@ Three complete example applications:
Comprehensive test coverage:
#### SDK Tests
- Client initialization validation
- API key format validation
- Configuration validation
- 4/4 tests passing
#### Integration Tests
- Public API documentation endpoint
- OpenAPI specification endpoint
- SDK information validation
@@ -139,6 +147,7 @@ backend/src/routes/publicApi.ts
### Type System
Complete type definitions for:
- Configuration (`SpywatcherConfig`)
- API Responses (`ApiResponse`, `PaginatedResponse`)
- User & Auth (`User`, `UserRole`, `ApiKeyInfo`)
@@ -150,6 +159,7 @@ Complete type definitions for:
The SDK covers all major Spywatcher API endpoints:
### Analytics
- ✅ Ghost users (`/ghosts`)
- ✅ Lurkers (`/lurkers`)
- ✅ Activity heatmap (`/heatmap`)
@@ -158,13 +168,16 @@ The SDK covers all major Spywatcher API endpoints:
- ✅ Status shifts (`/shifts`)
### Suspicion
- ✅ Suspicion data (`/suspicion`)
### Timeline
- ✅ Timeline events (`/timeline`)
- ✅ User timeline (`/timeline/:userId`)
### Bans
- ✅ List banned guilds (`/banned`)
- ✅ Ban guild (`/ban`)
- ✅ Unban guild (`/unban`)
@@ -173,25 +186,27 @@ The SDK covers all major Spywatcher API endpoints:
- ✅ Unban user (`/userunban`)
### Auth & User
- ✅ Current user (`/auth/me`)
- ✅ List API keys (`/auth/api-keys`)
- ✅ Create API key (`/auth/api-keys`)
- ✅ Revoke API key (`/auth/api-keys/:keyId`)
### Utility
- ✅ Health check (`/health`)
## Rate Limiting
Implemented rate limiting for public API endpoints:
| Endpoint Type | Limit | Window |
|--------------|-------|--------|
| Public API | 60 requests | 1 minute |
| Global API | 100 requests | 15 minutes |
| Analytics | 30 requests | 1 minute |
| Admin | 100 requests | 15 minutes |
| Authentication | 5 requests | 15 minutes |
| Endpoint Type | Limit | Window |
| -------------- | ------------ | ---------- |
| Public API | 60 requests | 1 minute |
| Global API | 100 requests | 15 minutes |
| Analytics | 30 requests | 1 minute |
| Admin | 100 requests | 15 minutes |
| Authentication | 5 requests | 15 minutes |
## Security
@@ -208,6 +223,7 @@ Security measures implemented:
## Build and Test Results
### SDK Build
```
✅ CommonJS build: 8.64 KB
✅ ES Module build: 6.77 KB
@@ -216,6 +232,7 @@ Security measures implemented:
```
### SDK Tests
```
✅ 4/4 tests passing
- API key format validation
@@ -225,6 +242,7 @@ Security measures implemented:
```
### Public API Tests
```
✅ 7/7 tests passing
- Documentation endpoint
@@ -243,8 +261,8 @@ Security measures implemented:
import { Spywatcher } from '@spywatcher/sdk';
const client = new Spywatcher({
baseUrl: 'https://api.spywatcher.com/api',
apiKey: process.env.SPYWATCHER_API_KEY!
baseUrl: 'https://api.spywatcher.com/api',
apiKey: process.env.SPYWATCHER_API_KEY!,
});
// Get ghost users
@@ -257,13 +275,13 @@ console.log(`Found ${ghosts.length} ghost users`);
```typescript
// Get activity patterns
const heatmap = await client.analytics.getHeatmap({
startDate: '2024-01-01',
endDate: '2024-12-31'
startDate: '2024-01-01',
endDate: '2024-12-31',
});
// Find peak activity hour
const peakHour = heatmap.reduce((max, curr) =>
curr.count > max.count ? curr : max
const peakHour = heatmap.reduce((max, curr) =>
curr.count > max.count ? curr : max
);
```
@@ -271,13 +289,13 @@ const peakHour = heatmap.reduce((max, curr) =>
```typescript
try {
const data = await client.analytics.getGhosts();
const data = await client.analytics.getGhosts();
} catch (error) {
if (error instanceof RateLimitError) {
console.error('Rate limit exceeded');
} else if (error instanceof AuthenticationError) {
console.error('Invalid API key');
}
if (error instanceof RateLimitError) {
console.error('Rate limit exceeded');
} else if (error instanceof AuthenticationError) {
console.error('Invalid API key');
}
}
```
@@ -320,6 +338,7 @@ When adding new API endpoints:
### Versioning
Follow semantic versioning:
- **Major**: Breaking API changes
- **Minor**: New features, backward compatible
- **Patch**: Bug fixes, backward compatible
@@ -334,7 +353,7 @@ Follow semantic versioning:
✅ **API fully documented**: JSON docs + OpenAPI spec
✅ **SDK published to npm**: Ready to publish (requires credentials)
✅ **Rate limiting enforced**: Multiple rate limit tiers
✅ **Example applications created**: 3 complete examples
✅ **Example applications created**: 3 complete examples
## Conclusion

11
SECURITY.md
View File

@@ -81,7 +81,10 @@ Strict CORS policy:
## 🚨 Reporting a Vulnerability
If you discover a security vulnerability, please email security@example.com (or create a private security advisory on GitHub).
If you discover a security vulnerability, please report it responsibly:
1. **Preferred Method**: Create a [private security advisory](https://github.com/subculture-collective/discord-spywatcher/security/advisories/new) on GitHub
2. **Alternative**: Contact the maintainers directly through GitHub
**Please do NOT create public issues for security vulnerabilities.**
@@ -104,9 +107,9 @@ If you discover a security vulnerability, please email security@example.com (or
1. Never commit `.env` files to version control
2. Use strong, randomly generated secrets for JWT keys:
```bash
node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
```
```bash
node -e "console.log(require('crypto').randomBytes(32).toString('hex'))"
```
3. Rotate secrets regularly (quarterly recommended)
4. Use different secrets for each environment (dev, staging, production)

22
SENTRY.md
View File

@@ -20,8 +20,8 @@ Sentry is integrated into both the frontend (React) and backend (Node.js/Express
1. Sign up at [sentry.io](https://sentry.io) or use your organization's Sentry instance
2. Create two projects:
- **Backend Project**: Node.js/Express platform
- **Frontend Project**: React platform
- **Backend Project**: Node.js/Express platform
- **Frontend Project**: React platform
3. Note the DSN (Data Source Name) for each project
### 2. Configure Environment Variables
@@ -66,9 +66,9 @@ For source map uploads:
1. Go to Sentry Settings > Auth Tokens
2. Create a new token with these scopes:
- `project:read`
- `project:releases`
- `org:read`
- `project:read`
- `project:releases`
- `org:read`
3. Copy the token to `VITE_SENTRY_AUTH_TOKEN`
## 📊 Features
@@ -357,13 +357,13 @@ export SENTRY_RELEASE="backend@1.0.0"
```yaml
- name: Deploy with Sentry
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
SENTRY_RELEASE: ${{ github.sha }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
SENTRY_RELEASE: ${{ github.sha }}
run: |
npm run build
# Source maps are automatically uploaded by the build
npm run build
# Source maps are automatically uploaded by the build
```
## 🧪 Testing

223
WEBSOCKET_API.md
View File

@@ -21,39 +21,42 @@ All WebSocket connections require JWT authentication. Include your access token
import { io } from 'socket.io-client';
const socket = io('http://localhost:3001', {
auth: {
token: 'your-jwt-access-token'
},
transports: ['websocket', 'polling']
auth: {
token: 'your-jwt-access-token',
},
transports: ['websocket', 'polling'],
});
```
### Connection Events
#### `connect`
Fired when the client successfully connects to the server.
```typescript
socket.on('connect', () => {
console.log('Connected to WebSocket server');
console.log('Connected to WebSocket server');
});
```
#### `disconnect`
Fired when the client disconnects from the server.
```typescript
socket.on('disconnect', (reason) => {
console.log('Disconnected:', reason);
console.log('Disconnected:', reason);
});
```
#### `connect_error`
Fired when a connection error occurs (e.g., authentication failure).
```typescript
socket.on('connect_error', (error) => {
console.error('Connection error:', error.message);
console.error('Connection error:', error.message);
});
```
@@ -70,6 +73,7 @@ socket.emit('subscribe:analytics', guildId);
```
**Parameters:**
- `guildId` (string): The Discord guild ID to subscribe to
#### Unsubscribe
@@ -79,6 +83,7 @@ socket.emit('unsubscribe:analytics', guildId);
```
**Parameters:**
- `guildId` (string): The Discord guild ID to unsubscribe from
### Guild Events Room
@@ -92,6 +97,7 @@ socket.emit('subscribe:guild', guildId);
```
**Parameters:**
- `guildId` (string): The Discord guild ID to subscribe to
#### Unsubscribe
@@ -101,6 +107,7 @@ socket.emit('unsubscribe:guild', guildId);
```
**Parameters:**
- `guildId` (string): The Discord guild ID to unsubscribe from
## Server Events
@@ -112,37 +119,39 @@ socket.emit('unsubscribe:guild', guildId);
Receives throttled analytics updates (maximum once per 30 seconds per guild).
**Event Data:**
```typescript
{
guildId: string;
data: {
ghosts: Array<{
userId: string;
username: string;
ghostScore: number;
}>;
lurkers: Array<{
userId: string;
username: string;
lurkerScore: number;
channelCount: number;
}>;
channelDiversity: Array<{
userId: string;
username: string;
channelCount: number;
}>;
guildId: string;
data: {
ghosts: Array<{
userId: string;
username: string;
ghostScore: number;
}>;
lurkers: Array<{
userId: string;
username: string;
lurkerScore: number;
channelCount: number;
}>;
channelDiversity: Array<{
userId: string;
username: string;
channelCount: number;
}>;
timestamp: string;
}
timestamp: string;
};
timestamp: string;
}
```
**Example:**
```typescript
socket.on('analytics:update', (data) => {
console.log('Analytics update:', data);
// Update your dashboard with new analytics
console.log('Analytics update:', data);
// Update your dashboard with new analytics
});
```
@@ -153,20 +162,22 @@ socket.on('analytics:update', (data) => {
Receives real-time notifications when a new message is created in the guild.
**Event Data:**
```typescript
{
userId: string;
username: string;
channelId: string;
channelName: string;
timestamp: string; // ISO 8601 format
userId: string;
username: string;
channelId: string;
channelName: string;
timestamp: string; // ISO 8601 format
}
```
**Example:**
```typescript
socket.on('message:new', (data) => {
console.log(`New message from ${data.username} in #${data.channelName}`);
console.log(`New message from ${data.username} in #${data.channelName}`);
});
```
@@ -175,6 +186,7 @@ socket.on('message:new', (data) => {
Receives alerts when a user is detected on multiple clients simultaneously.
**Event Data:**
```typescript
{
userId: string;
@@ -185,9 +197,12 @@ Receives alerts when a user is detected on multiple clients simultaneously.
```
**Example:**
```typescript
socket.on('alert:multiClient', (data) => {
console.warn(`Multi-client detected: ${data.username} on ${data.platforms.join(', ')}`);
console.warn(
`Multi-client detected: ${data.username} on ${data.platforms.join(', ')}`
);
});
```
@@ -196,19 +211,21 @@ socket.on('alert:multiClient', (data) => {
Receives real-time presence updates for guild members.
**Event Data:**
```typescript
{
userId: string;
username: string;
status: string; // 'online', 'idle', 'dnd', 'offline'
timestamp: string; // ISO 8601 format
userId: string;
username: string;
status: string; // 'online', 'idle', 'dnd', 'offline'
timestamp: string; // ISO 8601 format
}
```
**Example:**
```typescript
socket.on('presence:update', (data) => {
console.log(`${data.username} is now ${data.status}`);
console.log(`${data.username} is now ${data.status}`);
});
```
@@ -217,6 +234,7 @@ socket.on('presence:update', (data) => {
Receives notifications when a user's roles change in the guild.
**Event Data:**
```typescript
{
userId: string;
@@ -227,9 +245,10 @@ Receives notifications when a user's roles change in the guild.
```
**Example:**
```typescript
socket.on('role:change', (data) => {
console.log(`${data.username} gained roles: ${data.addedRoles.join(', ')}`);
console.log(`${data.username} gained roles: ${data.addedRoles.join(', ')}`);
});
```
@@ -238,19 +257,23 @@ socket.on('role:change', (data) => {
Receives notifications when a new user joins the guild.
**Event Data:**
```typescript
{
userId: string;
username: string;
accountAgeDays: number; // Age of the Discord account in days
timestamp: string; // ISO 8601 format
userId: string;
username: string;
accountAgeDays: number; // Age of the Discord account in days
timestamp: string; // ISO 8601 format
}
```
**Example:**
```typescript
socket.on('user:join', (data) => {
console.log(`${data.username} joined (account age: ${data.accountAgeDays} days)`);
console.log(
`${data.username} joined (account age: ${data.accountAgeDays} days)`
);
});
```
@@ -261,16 +284,18 @@ socket.on('user:join', (data) => {
Receives error messages from the server.
**Event Data:**
```typescript
{
message: string;
message: string;
}
```
**Example:**
```typescript
socket.on('error', (data) => {
console.error('Server error:', data.message);
console.error('Server error:', data.message);
});
```
@@ -285,11 +310,11 @@ socket.on('error', (data) => {
```typescript
// React example
useEffect(() => {
const socket = socketService.connect();
return () => {
socketService.disconnect();
};
const socket = socketService.connect();
return () => {
socketService.disconnect();
};
}, []);
```
@@ -301,13 +326,13 @@ useEffect(() => {
```typescript
useEffect(() => {
socketService.subscribeToGuild(guildId);
socketService.subscribeToAnalytics(guildId, handleAnalyticsUpdate);
return () => {
socketService.unsubscribeFromGuild(guildId);
socketService.unsubscribeFromAnalytics(guildId, handleAnalyticsUpdate);
};
socketService.subscribeToGuild(guildId);
socketService.subscribeToAnalytics(guildId, handleAnalyticsUpdate);
return () => {
socketService.unsubscribeFromGuild(guildId);
socketService.unsubscribeFromAnalytics(guildId, handleAnalyticsUpdate);
};
}, [guildId]);
```
@@ -320,7 +345,7 @@ useEffect(() => {
```typescript
// Good practice
const handleNewMessage = (data) => {
console.log('New message:', data);
console.log('New message:', data);
};
socket.on('message:new', handleNewMessage);
@@ -353,45 +378,45 @@ The server uses a Redis adapter for horizontal scaling, allowing multiple server
import { io, Socket } from 'socket.io-client';
class SocketService {
private socket: Socket | null = null;
private socket: Socket | null = null;
connect(token: string): Socket {
if (this.socket?.connected) {
return this.socket;
connect(token: string): Socket {
if (this.socket?.connected) {
return this.socket;
}
this.socket = io('http://localhost:3001', {
auth: { token },
transports: ['websocket', 'polling'],
reconnection: true,
reconnectionDelay: 1000,
reconnectionDelayMax: 5000,
});
this.socket.on('connect', () => {
console.log('WebSocket connected');
});
this.socket.on('disconnect', () => {
console.log('WebSocket disconnected');
});
return this.socket;
}
this.socket = io('http://localhost:3001', {
auth: { token },
transports: ['websocket', 'polling'],
reconnection: true,
reconnectionDelay: 1000,
reconnectionDelayMax: 5000,
});
subscribeToAnalytics(guildId: string, callback: (data: any) => void) {
if (!this.socket) return;
this.socket.on('connect', () => {
console.log('WebSocket connected');
});
this.socket.on('disconnect', () => {
console.log('WebSocket disconnected');
});
return this.socket;
}
subscribeToAnalytics(guildId: string, callback: (data: any) => void) {
if (!this.socket) return;
this.socket.emit('subscribe:analytics', guildId);
this.socket.on('analytics:update', callback);
}
disconnect() {
if (this.socket) {
this.socket.disconnect();
this.socket = null;
this.socket.emit('subscribe:analytics', guildId);
this.socket.on('analytics:update', callback);
}
disconnect() {
if (this.socket) {
this.socket.disconnect();
this.socket = null;
}
}
}
}
export const socketService = new SocketService();
@@ -452,7 +477,7 @@ export function LiveAnalyticsDashboard({ guildId, token }) {
</ul>
</div>
)}
<h3>Recent Messages</h3>
<ul>
{recentMessages.map((msg, i) => (
@@ -491,21 +516,25 @@ export function LiveAnalyticsDashboard({ guildId, token }) {
### Common Issues
#### Connection Refused
- Verify the WebSocket server is running
- Check that the port (3001) is not blocked by firewall
- Ensure CORS settings allow your origin
#### Authentication Failed
- Verify your JWT token is valid and not expired
- Check that the token includes required fields (discordId, access, role)
- Ensure the token is passed in the auth object during connection
#### Events Not Received
- Confirm you've subscribed to the correct room
- Check that event listeners are attached before events fire
- Verify the guild ID is correct
#### Multiple Connections
- Use a singleton pattern to ensure only one connection
- Check for duplicate connection attempts in your code
- Implement proper cleanup in component unmount