bugs(fix):Fixed broken data range filtering

surfsense_backend/alembic/versions/23_add_connector_schedules_table.py

@@ -0,0 +1,100 @@
+"""Add connector schedules table
+
+Revision ID: 23
+Revises: 22
+"""
+
+from collections.abc import Sequence
+
+from sqlalchemy import inspect
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "23"
+down_revision: str | None = "22"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Upgrade schema - add ScheduleType enum and connector_schedules table."""
+
+    # Create ScheduleType enum if it doesn't exist
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'scheduletype') THEN
+                CREATE TYPE scheduletype AS ENUM ('HOURLY', 'DAILY', 'WEEKLY', 'CUSTOM');
+            END IF;
+        END$$;
+    """
+    )
+
+    # Create connector_schedules table if it doesn't exist
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS connector_schedules (
+            id SERIAL PRIMARY KEY,
+            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+            connector_id INTEGER NOT NULL REFERENCES search_source_connectors(id) ON DELETE CASCADE,
+            search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE,
+            schedule_type scheduletype NOT NULL,
+            cron_expression VARCHAR(100),
+            daily_time TIME,
+            weekly_day SMALLINT,
+            weekly_time TIME,
+            hourly_minute SMALLINT,
+            is_active BOOLEAN NOT NULL DEFAULT TRUE,
+            last_run_at TIMESTAMPTZ,
+            next_run_at TIMESTAMPTZ,
+            CONSTRAINT uq_connector_search_space UNIQUE (connector_id, search_space_id)
+        );
+    """
+    )
+
+    # Get existing indexes
+    conn = op.get_bind()
+    inspector = inspect(conn)
+    existing_indexes = [
+        idx["name"] for idx in inspector.get_indexes("connector_schedules")
+    ]
+
+    # Create indexes only if they don't already exist
+    if "ix_connector_schedules_id" not in existing_indexes:
+        op.create_index("ix_connector_schedules_id", "connector_schedules", ["id"])
+    if "ix_connector_schedules_created_at" not in existing_indexes:
+        op.create_index(
+            "ix_connector_schedules_created_at", "connector_schedules", ["created_at"]
+        )
+    if "ix_connector_schedules_connector_id" not in existing_indexes:
+        op.create_index(
+            "ix_connector_schedules_connector_id", "connector_schedules", ["connector_id"]
+        )
+    if "ix_connector_schedules_is_active" not in existing_indexes:
+        op.create_index(
+            "ix_connector_schedules_is_active", "connector_schedules", ["is_active"]
+        )
+    if "ix_connector_schedules_next_run_at" not in existing_indexes:
+        op.create_index(
+            "ix_connector_schedules_next_run_at", "connector_schedules", ["next_run_at"]
+        )
+
+
+def downgrade() -> None:
+    """Downgrade schema - remove connector_schedules table and enum."""
+
+    # Drop indexes
+    op.drop_index("ix_connector_schedules_next_run_at", table_name="connector_schedules")
+    op.drop_index("ix_connector_schedules_is_active", table_name="connector_schedules")
+    op.drop_index("ix_connector_schedules_connector_id", table_name="connector_schedules")
+    op.drop_index("ix_connector_schedules_created_at", table_name="connector_schedules")
+    op.drop_index("ix_connector_schedules_id", table_name="connector_schedules")
+
+    # Drop table
+    op.drop_table("connector_schedules")
+
+    # Drop enum
+    op.execute("DROP TYPE IF EXISTS scheduletype")
+

surfsense_backend/app/app.py

@@ -1,4 +1,6 @@
-from contextlib import asynccontextmanager
+import asyncio
+import logging
+from contextlib import asynccontextmanager, suppress
 
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -7,15 +9,45 @@
 from app.config import config
 from app.db import User, create_db_and_tables, get_async_session
 from app.routes import router as crud_router
+from app.routes.connector_schedules_routes import router as connector_schedules_router
+from app.routes.scheduler_routes import router as scheduler_router
 from app.schemas import UserCreate, UserRead, UserUpdate
+from app.services.connector_scheduler_service import start_scheduler, stop_scheduler
 from app.users import SECRET, auth_backend, current_active_user, fastapi_users
 
+logger = logging.getLogger(__name__)
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # Not needed if you setup a migration system like Alembic
+    """Application lifespan manager with scheduler integration."""
+    # Startup
+    logger.info("Starting SurfSense application...")
+
+    # Create database tables
     await create_db_and_tables()
+    logger.info("Database tables created/verified")
+
+    # Start the connector scheduler service
+    scheduler_task = asyncio.create_task(start_scheduler())
+    logger.info("Connector scheduler service started")
+
     yield
+
+    # Shutdown
+    logger.info("Shutting down SurfSense application...")
+
+    # Stop the scheduler service
+    await stop_scheduler()
+    logger.info("Connector scheduler service stopped")
+
+    # Cancel the scheduler task
+    if not scheduler_task.done():
+        scheduler_task.cancel()
+        with suppress(asyncio.CancelledError):
+            await scheduler_task
+
+    logger.info("Application shutdown complete")
 
 
 app = FastAPI(lifespan=lifespan)
@@ -65,6 +97,8 @@ async def lifespan(app: FastAPI):
     )
 
 app.include_router(crud_router, prefix="/api/v1", tags=["crud"])
+app.include_router(connector_schedules_router, prefix="/api/v1", tags=["connector-schedules"])
+app.include_router(scheduler_router, prefix="/api/v1", tags=["scheduler"])
 
 
 @app.get("/verify-token")

surfsense_backend/app/connectors/clickup_connector.py

@@ -5,6 +5,7 @@
 Allows fetching tasks from workspaces and lists.
 """
 
+from datetime import datetime
 from typing import Any
 
 import requests
@@ -168,13 +169,28 @@ def get_tasks_in_date_range(
             Tuple containing (tasks list, error message or None)
         """
         try:
-            # TODO : Include date range in api request
+            # Convert date strings to Unix timestamps (milliseconds)
+            start_datetime = datetime.strptime(start_date, "%Y-%m-%d")
+            end_datetime = datetime.strptime(end_date, "%Y-%m-%d")
+
+            # Set time to start and end of day for complete coverage
+            start_datetime = start_datetime.replace(hour=0, minute=0, second=0, microsecond=0)
+            end_datetime = end_datetime.replace(hour=23, minute=59, second=59, microsecond=999999)
+
+            start_timestamp = int(start_datetime.timestamp() * 1000)
+            end_timestamp = int(end_datetime.timestamp() * 1000)
 
             params = {
                 "page": 0,
                 "order_by": "created",
                 "reverse": "true",
                 "subtasks": "true",
+                "include_closed": str(include_closed).lower(),
+                # Date filtering - filter by both created and updated dates
+                "date_created_gt": start_timestamp,
+                "date_created_lt": end_timestamp,
+                "date_updated_gt": start_timestamp,
+                "date_updated_lt": end_timestamp,
             }
 
             all_tasks = []

surfsense_backend/app/connectors/jira_connector.py

@@ -222,14 +222,14 @@ def get_issues_by_date_range(
             # Build JQL query for date range
             # Query issues that were either created OR updated within the date range
             date_filter = (
-                f"(createdDate >= '{start_date}' AND createdDate <= '{end_date}')"
+                f"(createdDate >= '{start_date}' AND createdDate <= '{end_date}') "
+                f"OR (updatedDate >= '{start_date}' AND updatedDate <= '{end_date}')"
             )
-            # TODO : This JQL needs some improvement to work as expected
 
-            _jql = f"{date_filter}"
+            _jql = f"{date_filter} ORDER BY created DESC"
             if project_key:
                 _jql = (
-                    f'project = "{project_key}" AND {date_filter} ORDER BY created DESC'
+                    f'project = "{project_key}" AND ({date_filter}) ORDER BY created DESC'
                 )
 
             # Define fields to retrieve
@@ -250,7 +250,7 @@ def get_issues_by_date_range(
                 fields.append("comment")
 
             params = {
-                # "jql": "",   TODO : Add a JQL query to filter from a date range
+                "jql": _jql,
                 "fields": ",".join(fields),
                 "maxResults": 100,
                 "startAt": 0,

surfsense_backend/app/db.py

@@ -16,6 +16,7 @@
     Integer,
     String,
     Text,
+    Time,
     UniqueConstraint,
     text,
 )
@@ -129,6 +130,13 @@ class LogStatus(str, Enum):
     FAILED = "FAILED"
 
 
+class ScheduleType(str, Enum):
+    HOURLY = "HOURLY"
+    DAILY = "DAILY"
+    WEEKLY = "WEEKLY"
+    CUSTOM = "CUSTOM"
+
+
 class Base(DeclarativeBase):
     pass
 
@@ -245,21 +253,10 @@ class SearchSpace(BaseModel, TimestampMixin):
         order_by="Log.id",
         cascade="all, delete-orphan",
     )
-    search_source_connectors = relationship(
-        "SearchSourceConnector",
-        back_populates="search_space",
-        order_by="SearchSourceConnector.id",
-        cascade="all, delete-orphan",
-    )
-    llm_configs = relationship(
-        "LLMConfig",
-        back_populates="search_space",
-        order_by="LLMConfig.id",
-        cascade="all, delete-orphan",
-    )
-    user_preferences = relationship(
-        "UserSearchSpacePreference",
+    connector_schedules = relationship(
+        "ConnectorSchedule",
         back_populates="search_space",
+        order_by="ConnectorSchedule.id",
         cascade="all, delete-orphan",
     )
 
@@ -291,6 +288,46 @@ class SearchSourceConnector(BaseModel, TimestampMixin):
     user_id = Column(
         UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
     )
+    user = relationship("User", back_populates="search_source_connectors")
+    schedules = relationship(
+        "ConnectorSchedule",
+        back_populates="connector",
+        cascade="all, delete-orphan",
+    )
+
+
+class ConnectorSchedule(BaseModel, TimestampMixin):
+    __tablename__ = "connector_schedules"
+    __table_args__ = (
+        UniqueConstraint(
+            "connector_id", "search_space_id", name="uq_connector_search_space"
+        ),
+    )
+
+    connector_id = Column(
+        Integer,
+        ForeignKey("search_source_connectors.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    search_space_id = Column(
+        Integer,
+        ForeignKey("searchspaces.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    schedule_type = Column(SQLAlchemyEnum(ScheduleType), nullable=False)
+    cron_expression = Column(String(100), nullable=True)
+    # Optional schedule config fields (persist user selections)
+    daily_time = Column(Time(timezone=False), nullable=True)
+    weekly_day = Column(Integer, nullable=True)  # 0=Mon .. 6=Sun
+    weekly_time = Column(Time(timezone=False), nullable=True)
+    hourly_minute = Column(Integer, nullable=True)  # 0..59
+    is_active = Column(Boolean, nullable=False, default=True, index=True)
+    last_run_at = Column(TIMESTAMP(timezone=True), nullable=True)
+    next_run_at = Column(TIMESTAMP(timezone=True), nullable=True, index=True)
+
+    connector = relationship("SearchSourceConnector", back_populates="schedules")
+    search_space = relationship("SearchSpace", back_populates="connector_schedules")
 
 
 class LLMConfig(BaseModel, TimestampMixin):