Добавил форму для загрузки данных с LyngSat
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
# This will make sure the app is always imported when
|
||||
# Django starts so that shared_task will use this app.
|
||||
try:
|
||||
from .celery import app as celery_app
|
||||
__all__ = ('celery_app',)
|
||||
except ImportError:
|
||||
# Celery is not installed, skip initialization
|
||||
pass
|
||||
|
||||
24
dbapp/dbapp/celery.py
Normal file
24
dbapp/dbapp/celery.py
Normal file
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Celery configuration for dbapp project.
|
||||
"""
|
||||
import os
|
||||
from celery import Celery
|
||||
|
||||
# Set the default Django settings module for the 'celery' program.
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dbapp.settings.development')
|
||||
|
||||
app = Celery('dbapp')
|
||||
|
||||
# Using a string here means the worker doesn't have to serialize
|
||||
# the configuration object to child processes.
|
||||
# - namespace='CELERY' means all celery-related configuration keys
|
||||
# should have a `CELERY_` prefix.
|
||||
app.config_from_object('django.conf:settings', namespace='CELERY')
|
||||
|
||||
# Load task modules from all registered Django apps.
|
||||
app.autodiscover_tasks()
|
||||
|
||||
|
||||
@app.task(bind=True, ignore_result=True)
|
||||
def debug_task(self):
|
||||
print(f'Request: {self.request!r}')
|
||||
@@ -21,19 +21,21 @@ load_dotenv()
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
# GDAL/GEOS configuration for Windows
|
||||
if os.name == 'nt':
|
||||
if os.name == "nt":
|
||||
OSGEO4W = r"C:\Program Files\OSGeo4W"
|
||||
assert os.path.isdir(OSGEO4W), "Directory does not exist: " + OSGEO4W
|
||||
os.environ['OSGEO4W_ROOT'] = OSGEO4W
|
||||
os.environ['PROJ_LIB'] = os.path.join(OSGEO4W, r"share\proj")
|
||||
os.environ['PATH'] = OSGEO4W + r"\bin;" + os.environ['PATH']
|
||||
os.environ["OSGEO4W_ROOT"] = OSGEO4W
|
||||
os.environ["PROJ_LIB"] = os.path.join(OSGEO4W, r"share\proj")
|
||||
os.environ["PATH"] = OSGEO4W + r"\bin;" + os.environ["PATH"]
|
||||
|
||||
# ============================================================================
|
||||
# SECURITY SETTINGS
|
||||
# ============================================================================
|
||||
|
||||
# SECURITY WARNING: keep the secret key used in production secret!
|
||||
SECRET_KEY = os.getenv('SECRET_KEY', 'django-insecure-7etj5f7buo2a57xv=w3^&llusq8rii7b_gd)9$t_1xcnao!^tq')
|
||||
SECRET_KEY = os.getenv(
|
||||
"SECRET_KEY", "django-insecure-7etj5f7buo2a57xv=w3^&llusq8rii7b_gd)9$t_1xcnao!^tq"
|
||||
)
|
||||
|
||||
# SECURITY WARNING: don't run with debug turned on in production!
|
||||
# This should be overridden in environment-specific settings
|
||||
@@ -49,38 +51,42 @@ ALLOWED_HOSTS = []
|
||||
|
||||
INSTALLED_APPS = [
|
||||
# Django Autocomplete Light (must be before admin)
|
||||
'dal',
|
||||
'dal_select2',
|
||||
|
||||
"dal",
|
||||
"dal_select2",
|
||||
# Admin interface customization
|
||||
'admin_interface',
|
||||
'colorfield',
|
||||
|
||||
"admin_interface",
|
||||
"colorfield",
|
||||
# Django GIS
|
||||
'django.contrib.gis',
|
||||
|
||||
"django.contrib.gis",
|
||||
# Django core apps
|
||||
'django.contrib.admin',
|
||||
'django.contrib.auth',
|
||||
'django.contrib.contenttypes',
|
||||
'django.contrib.sessions',
|
||||
'django.contrib.messages',
|
||||
'django.contrib.staticfiles',
|
||||
'django.contrib.humanize',
|
||||
|
||||
"django.contrib.admin",
|
||||
"django.contrib.auth",
|
||||
"django.contrib.contenttypes",
|
||||
"django.contrib.sessions",
|
||||
"django.contrib.messages",
|
||||
"django.contrib.staticfiles",
|
||||
"django.contrib.humanize",
|
||||
# Third-party apps
|
||||
'leaflet',
|
||||
'dynamic_raw_id',
|
||||
'rangefilter',
|
||||
'django_admin_multiple_choice_list_filter',
|
||||
'more_admin_filters',
|
||||
'import_export',
|
||||
|
||||
"leaflet",
|
||||
"dynamic_raw_id",
|
||||
"rangefilter",
|
||||
"django_admin_multiple_choice_list_filter",
|
||||
"more_admin_filters",
|
||||
"import_export",
|
||||
# Project apps
|
||||
'mainapp',
|
||||
'mapsapp',
|
||||
"mainapp",
|
||||
"mapsapp",
|
||||
"lyngsatapp",
|
||||
]
|
||||
|
||||
# Add Celery results app if available
|
||||
try:
|
||||
import django_celery_results
|
||||
|
||||
INSTALLED_APPS.append("django_celery_results")
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Note: Custom user model is implemented via OneToOneField relationship
|
||||
# If you need a custom user model, uncomment and configure:
|
||||
# AUTH_USER_MODEL = 'mainapp.CustomUser'
|
||||
@@ -90,17 +96,17 @@ INSTALLED_APPS = [
|
||||
# ============================================================================
|
||||
|
||||
MIDDLEWARE = [
|
||||
'django.middleware.security.SecurityMiddleware',
|
||||
'django.contrib.sessions.middleware.SessionMiddleware',
|
||||
'django.middleware.locale.LocaleMiddleware',
|
||||
'django.middleware.common.CommonMiddleware',
|
||||
'django.middleware.csrf.CsrfViewMiddleware',
|
||||
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
||||
'django.contrib.messages.middleware.MessageMiddleware',
|
||||
'django.middleware.clickjacking.XFrameOptionsMiddleware',
|
||||
"django.middleware.security.SecurityMiddleware",
|
||||
"django.contrib.sessions.middleware.SessionMiddleware",
|
||||
"django.middleware.locale.LocaleMiddleware",
|
||||
"django.middleware.common.CommonMiddleware",
|
||||
"django.middleware.csrf.CsrfViewMiddleware",
|
||||
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
||||
"django.contrib.messages.middleware.MessageMiddleware",
|
||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||
]
|
||||
|
||||
ROOT_URLCONF = 'dbapp.urls'
|
||||
ROOT_URLCONF = "dbapp.urls"
|
||||
|
||||
# ============================================================================
|
||||
# TEMPLATES CONFIGURATION
|
||||
@@ -108,36 +114,36 @@ ROOT_URLCONF = 'dbapp.urls'
|
||||
|
||||
TEMPLATES = [
|
||||
{
|
||||
'BACKEND': 'django.template.backends.django.DjangoTemplates',
|
||||
'DIRS': [
|
||||
BASE_DIR / 'templates',
|
||||
"BACKEND": "django.template.backends.django.DjangoTemplates",
|
||||
"DIRS": [
|
||||
BASE_DIR / "templates",
|
||||
],
|
||||
'APP_DIRS': True,
|
||||
'OPTIONS': {
|
||||
'context_processors': [
|
||||
'django.template.context_processors.debug',
|
||||
'django.template.context_processors.request',
|
||||
'django.contrib.auth.context_processors.auth',
|
||||
'django.contrib.messages.context_processors.messages',
|
||||
"APP_DIRS": True,
|
||||
"OPTIONS": {
|
||||
"context_processors": [
|
||||
"django.template.context_processors.debug",
|
||||
"django.template.context_processors.request",
|
||||
"django.contrib.auth.context_processors.auth",
|
||||
"django.contrib.messages.context_processors.messages",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
WSGI_APPLICATION = 'dbapp.wsgi.application'
|
||||
WSGI_APPLICATION = "dbapp.wsgi.application"
|
||||
|
||||
# ============================================================================
|
||||
# DATABASE CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
DATABASES = {
|
||||
'default': {
|
||||
'ENGINE': os.getenv('DB_ENGINE', 'django.contrib.gis.db.backends.postgis'),
|
||||
'NAME': os.getenv('DB_NAME', 'db'),
|
||||
'USER': os.getenv('DB_USER', 'user'),
|
||||
'PASSWORD': os.getenv('DB_PASSWORD', 'password'),
|
||||
'HOST': os.getenv('DB_HOST', 'localhost'),
|
||||
'PORT': os.getenv('DB_PORT', '5432'),
|
||||
"default": {
|
||||
"ENGINE": os.getenv("DB_ENGINE", "django.contrib.gis.db.backends.postgis"),
|
||||
"NAME": os.getenv("DB_NAME", "db"),
|
||||
"USER": os.getenv("DB_USER", "user"),
|
||||
"PASSWORD": os.getenv("DB_PASSWORD", "password"),
|
||||
"HOST": os.getenv("DB_HOST", "localhost"),
|
||||
"PORT": os.getenv("DB_PORT", "5432"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,16 +154,16 @@ DATABASES = {
|
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
|
||||
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
|
||||
},
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
|
||||
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
|
||||
},
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
|
||||
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
|
||||
},
|
||||
{
|
||||
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
|
||||
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
|
||||
},
|
||||
]
|
||||
|
||||
@@ -165,9 +171,9 @@ AUTH_PASSWORD_VALIDATORS = [
|
||||
# INTERNATIONALIZATION
|
||||
# ============================================================================
|
||||
|
||||
LANGUAGE_CODE = 'ru'
|
||||
LANGUAGE_CODE = "ru"
|
||||
|
||||
TIME_ZONE = 'Europe/Moscow'
|
||||
TIME_ZONE = "Europe/Moscow"
|
||||
|
||||
USE_I18N = True
|
||||
|
||||
@@ -177,18 +183,18 @@ USE_TZ = True
|
||||
# AUTHENTICATION CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
LOGIN_URL = 'login'
|
||||
LOGIN_REDIRECT_URL = 'mainapp:home'
|
||||
LOGOUT_REDIRECT_URL = 'mainapp:home'
|
||||
LOGIN_URL = "login"
|
||||
LOGIN_REDIRECT_URL = "mainapp:home"
|
||||
LOGOUT_REDIRECT_URL = "mainapp:home"
|
||||
|
||||
# ============================================================================
|
||||
# STATIC FILES CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
STATIC_URL = '/static/'
|
||||
STATIC_URL = "/static/"
|
||||
|
||||
STATICFILES_DIRS = [
|
||||
BASE_DIR.parent / 'static',
|
||||
BASE_DIR.parent / "static",
|
||||
]
|
||||
|
||||
# STATIC_ROOT will be set in production.py
|
||||
@@ -198,7 +204,7 @@ STATICFILES_DIRS = [
|
||||
# ============================================================================
|
||||
|
||||
# Default primary key field type
|
||||
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||
|
||||
# ============================================================================
|
||||
# THIRD-PARTY APP CONFIGURATION
|
||||
@@ -210,17 +216,53 @@ SILENCED_SYSTEM_CHECKS = ["security.W019"]
|
||||
|
||||
# Leaflet Configuration
|
||||
LEAFLET_CONFIG = {
|
||||
'ATTRIBUTION_PREFIX': '',
|
||||
'TILES': [
|
||||
"ATTRIBUTION_PREFIX": "",
|
||||
"TILES": [
|
||||
(
|
||||
'Satellite',
|
||||
'https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}',
|
||||
{'attribution': '© Esri', 'maxZoom': 16}
|
||||
"Satellite",
|
||||
"https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}",
|
||||
{"attribution": "© Esri", "maxZoom": 16},
|
||||
),
|
||||
(
|
||||
'Streets',
|
||||
'http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
|
||||
{'attribution': '© <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'}
|
||||
)
|
||||
"Streets",
|
||||
"http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",
|
||||
{
|
||||
"attribution": '© <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
|
||||
},
|
||||
),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# CELERY CONFIGURATION
|
||||
# ============================================================================
|
||||
|
||||
# Celery Configuration Options
|
||||
CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
|
||||
CELERY_RESULT_BACKEND = "django-db"
|
||||
CELERY_CACHE_BACKEND = "default"
|
||||
|
||||
# Celery Task Configuration
|
||||
CELERY_TASK_TRACK_STARTED = True
|
||||
CELERY_TASK_TIME_LIMIT = 30 * 60 # 30 minutes
|
||||
CELERY_TASK_SOFT_TIME_LIMIT = 25 * 60 # 25 minutes
|
||||
CELERY_TASK_ALWAYS_EAGER = False # Set to True for synchronous execution in development
|
||||
|
||||
# Celery Beat Configuration (for periodic tasks)
|
||||
CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
|
||||
|
||||
# Celery Result Backend Configuration
|
||||
CELERY_RESULT_EXTENDED = True
|
||||
CELERY_RESULT_EXPIRES = 3600 # Results expire after 1 hour
|
||||
|
||||
# Celery Logging
|
||||
CELERY_WORKER_HIJACK_ROOT_LOGGER = False
|
||||
CELERY_WORKER_LOG_FORMAT = "[%(asctime)s: %(levelname)s/%(processName)s] %(message)s"
|
||||
CELERY_WORKER_TASK_LOG_FORMAT = "[%(asctime)s: %(levelname)s/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s"
|
||||
|
||||
# Celery Accept Content
|
||||
CELERY_ACCEPT_CONTENT = ["json"]
|
||||
CELERY_TASK_SERIALIZER = "json"
|
||||
CELERY_RESULT_SERIALIZER = "json"
|
||||
CELERY_TIMEZONE = TIME_ZONE
|
||||
|
||||
@@ -3,6 +3,8 @@ from .models import LyngSat
|
||||
|
||||
@admin.register(LyngSat)
|
||||
class LyngSatAdmin(admin.ModelAdmin):
|
||||
list_display = ("mark", "timestamp")
|
||||
search_fields = ("mark", )
|
||||
ordering = ("timestamp",)
|
||||
list_display = ("id_satellite", "frequency", "polarization", "modulation", "last_update")
|
||||
search_fields = ("id_satellite__name", "channel_info")
|
||||
list_filter = ("id_satellite", "polarization", "modulation", "standard")
|
||||
ordering = ("-last_update",)
|
||||
readonly_fields = ("last_update",)
|
||||
37
dbapp/lyngsatapp/migrations/0001_initial.py
Normal file
37
dbapp/lyngsatapp/migrations/0001_initial.py
Normal file
@@ -0,0 +1,37 @@
|
||||
# Generated by Django 5.2.7 on 2025-11-10 20:03
|
||||
|
||||
import django.db.models.deletion
|
||||
import mainapp.models
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
('mainapp', '0007_remove_parameter_objitems_parameter_objitem'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='LyngSat',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('frequency', models.FloatField(blank=True, default=0, null=True, verbose_name='Частота, МГц')),
|
||||
('sym_velocity', models.FloatField(blank=True, default=0, null=True, verbose_name='Символьная скорость, БОД')),
|
||||
('last_update', models.DateTimeField(blank=True, null=True, verbose_name='Время')),
|
||||
('channel_info', models.CharField(blank=True, max_length=20, null=True, verbose_name='Описание источника')),
|
||||
('fec', models.CharField(blank=True, max_length=30, null=True, verbose_name='Коэффициент коррекции ошибок')),
|
||||
('url', models.URLField(blank=True, null=True, verbose_name='Ссылка на страницу')),
|
||||
('id_satellite', models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='lyngsat', to='mainapp.satellite', verbose_name='Спутник')),
|
||||
('modulation', models.ForeignKey(blank=True, default=mainapp.models.get_default_modulation, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='lyngsat', to='mainapp.modulation', verbose_name='Модуляция')),
|
||||
('polarization', models.ForeignKey(blank=True, default=mainapp.models.get_default_polarization, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='lyngsat', to='mainapp.polarization', verbose_name='Поляризация')),
|
||||
('standard', models.ForeignKey(blank=True, default=mainapp.models.get_default_standard, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='lyngsat', to='mainapp.standard', verbose_name='Стандарт')),
|
||||
],
|
||||
options={
|
||||
'verbose_name': 'Источник LyngSat',
|
||||
'verbose_name_plural': 'Источники LyngSat',
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -4,76 +4,80 @@ from datetime import datetime
|
||||
import re
|
||||
import time
|
||||
|
||||
|
||||
class LyngSatParser:
|
||||
"""Парсер данных для LyngSat(Для работы нужен flaresolver)"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
self,
|
||||
flaresolver_url: str = "http://localhost:8191/v1",
|
||||
regions: list[str] | None = None,
|
||||
target_sats: list[str] | None = None,
|
||||
):
|
||||
self.flaresolver_url = flaresolver_url
|
||||
self.regions = regions
|
||||
self.target_sats = list(map(lambda sat: sat.strip().lower(), target_sats)) if regions else None
|
||||
self.regions = regions if regions else ["europe", "asia", "america", "atlantic"]
|
||||
self.target_sats = (
|
||||
list(map(lambda sat: sat.strip().lower(), target_sats)) if regions else None
|
||||
)
|
||||
self.regions = regions if regions else ["europe", "asia", "america", "atlantic"]
|
||||
self.BASE_URL = "https://www.lyngsat.com"
|
||||
|
||||
|
||||
def parse_metadata(self, metadata: str) -> dict:
|
||||
if not metadata or not metadata.strip():
|
||||
return {
|
||||
'standard': None,
|
||||
'modulation': None,
|
||||
'symbol_rate': None,
|
||||
'fec': None
|
||||
"standard": None,
|
||||
"modulation": None,
|
||||
"symbol_rate": None,
|
||||
"fec": None,
|
||||
}
|
||||
normalized = re.sub(r'\s+', '', metadata.strip())
|
||||
fec_match = re.search(r'([1-9]/[1-9])$', normalized)
|
||||
normalized = re.sub(r"\s+", "", metadata.strip())
|
||||
fec_match = re.search(r"([1-9]/[1-9])$", normalized)
|
||||
fec = fec_match.group(1) if fec_match else None
|
||||
if fec_match:
|
||||
core = normalized[:fec_match.start()]
|
||||
core = normalized[: fec_match.start()]
|
||||
else:
|
||||
core = normalized
|
||||
std_match = re.match(r'(DVB-S2?|ABS-S|DVB-T2?|ATSC|ISDB)', core)
|
||||
std_match = re.match(r"(DVB-S2?|ABS-S|DVB-T2?|ATSC|ISDB)", core)
|
||||
standard = std_match.group(1) if std_match else None
|
||||
rest = core[len(standard):] if standard else core
|
||||
rest = core[len(standard) :] if standard else core
|
||||
modulation = None
|
||||
mod_match = re.match(r'(8PSK|QPSK|16APSK|32APSK|64QAM|256QAM|BPSK)', rest)
|
||||
mod_match = re.match(r"(8PSK|QPSK|16APSK|32APSK|64QAM|256QAM|BPSK)", rest)
|
||||
if mod_match:
|
||||
modulation = mod_match.group(1)
|
||||
rest = rest[len(modulation):]
|
||||
rest = rest[len(modulation) :]
|
||||
symbol_rate = None
|
||||
sr_match = re.search(r'(\d+)$', rest)
|
||||
sr_match = re.search(r"(\d+)$", rest)
|
||||
if sr_match:
|
||||
try:
|
||||
symbol_rate = int(sr_match.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
return {
|
||||
'standard': standard,
|
||||
'modulation': modulation,
|
||||
'symbol_rate': symbol_rate,
|
||||
'fec': fec
|
||||
"standard": standard,
|
||||
"modulation": modulation,
|
||||
"symbol_rate": symbol_rate,
|
||||
"fec": fec,
|
||||
}
|
||||
|
||||
def extract_date(self, s: str) -> datetime | None:
|
||||
s = s.strip()
|
||||
match = re.search(r'(\d{6})$', s)
|
||||
match = re.search(r"(\d{6})$", s)
|
||||
if not match:
|
||||
return None
|
||||
yymmdd = match.group(1)
|
||||
try:
|
||||
return datetime.strptime(yymmdd, '%y%m%d').date()
|
||||
return datetime.strptime(yymmdd, "%y%m%d").date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def convert_polarization(self, polarization: str) -> str:
|
||||
"""Преобразовать код поляризации в понятное название на русском"""
|
||||
polarization_map = {
|
||||
'V': 'Вертикальная',
|
||||
'H': 'Горизонтальная',
|
||||
'R': 'Правая',
|
||||
'L': 'Левая'
|
||||
"V": "Вертикальная",
|
||||
"H": "Горизонтальная",
|
||||
"R": "Правая",
|
||||
"L": "Левая",
|
||||
}
|
||||
return polarization_map.get(polarization.upper(), polarization)
|
||||
|
||||
@@ -83,11 +87,7 @@ class LyngSatParser:
|
||||
regions = self.regions
|
||||
for region in regions:
|
||||
url = f"{self.BASE_URL}/{region}.html"
|
||||
payload = {
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": 60000
|
||||
}
|
||||
payload = {"cmd": "request.get", "url": url, "maxTimeout": 60000}
|
||||
response = requests.post(self.flaresolver_url, json=payload)
|
||||
if response.status_code != 200:
|
||||
continue
|
||||
@@ -95,7 +95,7 @@ class LyngSatParser:
|
||||
html_regions.append(html_content)
|
||||
print(f"Обработал страницу по {region}")
|
||||
return html_regions
|
||||
|
||||
|
||||
def get_satellite_urls(self, html_regions: list[str]):
|
||||
sat_names = []
|
||||
sat_urls = []
|
||||
@@ -104,19 +104,19 @@ class LyngSatParser:
|
||||
|
||||
col_table = soup.find_all("div", class_="desktab")[0]
|
||||
|
||||
tables = col_table.find_next_sibling('table').find_all('table')
|
||||
tables = col_table.find_next_sibling("table").find_all("table")
|
||||
trs = []
|
||||
for table in tables:
|
||||
trs.extend(table.find_all('tr'))
|
||||
trs.extend(table.find_all("tr"))
|
||||
for tr in trs:
|
||||
sat_name = tr.find('span').text
|
||||
sat_name = tr.find("span").text
|
||||
if self.target_sats is not None:
|
||||
if sat_name.strip().lower() not in self.target_sats:
|
||||
continue
|
||||
try:
|
||||
sat_url = tr.find_all('a')[2]['href']
|
||||
sat_url = tr.find_all("a")[2]["href"]
|
||||
except IndexError:
|
||||
sat_url = tr.find_all('a')[0]['href']
|
||||
sat_url = tr.find_all("a")[0]["href"]
|
||||
sat_names.append(sat_name)
|
||||
sat_urls.append(sat_url)
|
||||
return sat_names, sat_urls
|
||||
@@ -128,60 +128,67 @@ class LyngSatParser:
|
||||
|
||||
col_table = soup.find_all("div", class_="desktab")[0]
|
||||
|
||||
tables = col_table.find_next_sibling('table').find_all('table')
|
||||
tables = col_table.find_next_sibling("table").find_all("table")
|
||||
trs = []
|
||||
for table in tables:
|
||||
trs.extend(table.find_all('tr'))
|
||||
trs.extend(table.find_all("tr"))
|
||||
for tr in trs:
|
||||
sat_name = tr.find('span').text
|
||||
sat_name = tr.find("span").text
|
||||
if self.target_sats is not None:
|
||||
if sat_name.strip().lower() not in self.target_sats:
|
||||
continue
|
||||
try:
|
||||
sat_url = tr.find_all('a')[2]['href']
|
||||
sat_url = tr.find_all("a")[2]["href"]
|
||||
except IndexError:
|
||||
sat_url = tr.find_all('a')[0]['href']
|
||||
|
||||
update_date = tr.find_all('td')[-1].text
|
||||
sat_response = requests.post(self.flaresolver_url, json={
|
||||
"cmd": "request.get",
|
||||
"url": f"{self.BASE_URL}/{sat_url}",
|
||||
"maxTimeout": 60000
|
||||
})
|
||||
html_content = sat_response.json().get("solution", {}).get("response", "")
|
||||
sat_url = tr.find_all("a")[0]["href"]
|
||||
|
||||
update_date = tr.find_all("td")[-1].text
|
||||
sat_response = requests.post(
|
||||
self.flaresolver_url,
|
||||
json={
|
||||
"cmd": "request.get",
|
||||
"url": f"{self.BASE_URL}/{sat_url}",
|
||||
"maxTimeout": 60000,
|
||||
},
|
||||
)
|
||||
html_content = (
|
||||
sat_response.json().get("solution", {}).get("response", "")
|
||||
)
|
||||
sat_page_data = self.get_satellite_content(html_content)
|
||||
sat_data[sat_name] = {
|
||||
"url": f"{self.BASE_URL}/{sat_url}",
|
||||
"update_date": datetime.strptime(update_date, "%y%m%d").date(),
|
||||
"sources": sat_page_data
|
||||
"sources": sat_page_data,
|
||||
}
|
||||
return sat_data
|
||||
|
||||
|
||||
def get_satellite_content(self, html_content: str) -> dict:
|
||||
sat_soup = BeautifulSoup(html_content, "html.parser")
|
||||
big_table = sat_soup.find('table', class_='bigtable')
|
||||
big_table = sat_soup.find("table", class_="bigtable")
|
||||
all_tables = big_table.find_all("div", class_="desktab")[:-1]
|
||||
data = []
|
||||
for table in all_tables:
|
||||
trs = table.find_next_sibling('table').find_all('tr')
|
||||
trs = table.find_next_sibling("table").find_all("tr")
|
||||
for idx, tr in enumerate(trs):
|
||||
tds = tr.find_all('td')
|
||||
tds = tr.find_all("td")
|
||||
if len(tds) < 9 or idx < 2:
|
||||
continue
|
||||
freq, polarization = tds[0].find('b').text.strip().split('\xa0')
|
||||
freq, polarization = tds[0].find("b").text.strip().split("\xa0")
|
||||
polarization = self.convert_polarization(polarization)
|
||||
meta = self.parse_metadata(tds[1].text)
|
||||
provider_name = tds[3].text
|
||||
last_update = self.extract_date(tds[-1].text)
|
||||
data.append({
|
||||
"freq": freq,
|
||||
"pol": polarization,
|
||||
"metadata": meta,
|
||||
"provider_name": provider_name,
|
||||
"last_update": last_update
|
||||
})
|
||||
data.append(
|
||||
{
|
||||
"freq": freq,
|
||||
"pol": polarization,
|
||||
"metadata": meta,
|
||||
"provider_name": provider_name,
|
||||
"last_update": last_update,
|
||||
}
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
|
||||
class KingOfSatParser:
|
||||
def __init__(self, base_url="https://ru.kingofsat.net", max_satellites=0):
|
||||
@@ -193,20 +200,22 @@ class KingOfSatParser:
|
||||
self.base_url = base_url
|
||||
self.max_satellites = max_satellites
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
||||
})
|
||||
|
||||
self.session.headers.update(
|
||||
{
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
}
|
||||
)
|
||||
|
||||
def convert_polarization(self, polarization):
|
||||
"""Преобразовать код поляризации в понятное название на русском"""
|
||||
polarization_map = {
|
||||
'V': 'Вертикальная',
|
||||
'H': 'Горизонтальная',
|
||||
'R': 'Правая',
|
||||
'L': 'Левая'
|
||||
"V": "Вертикальная",
|
||||
"H": "Горизонтальная",
|
||||
"R": "Правая",
|
||||
"L": "Левая",
|
||||
}
|
||||
return polarization_map.get(polarization.upper(), polarization)
|
||||
|
||||
|
||||
def fetch_page(self, url):
|
||||
"""Получить HTML страницу"""
|
||||
try:
|
||||
@@ -216,114 +225,112 @@ class KingOfSatParser:
|
||||
except Exception as e:
|
||||
print(f"Ошибка при получении страницы {url}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_satellite_table(self, html_content):
|
||||
"""Распарсить таблицу со спутниками"""
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
soup = BeautifulSoup(html_content, "html.parser")
|
||||
satellites = []
|
||||
table = soup.find('table')
|
||||
table = soup.find("table")
|
||||
if not table:
|
||||
print("Таблица не найдена")
|
||||
return satellites
|
||||
|
||||
rows = table.find_all('tr')[1:]
|
||||
|
||||
|
||||
rows = table.find_all("tr")[1:]
|
||||
|
||||
for row in rows:
|
||||
cols = row.find_all('td')
|
||||
cols = row.find_all("td")
|
||||
if len(cols) < 13:
|
||||
continue
|
||||
|
||||
|
||||
try:
|
||||
position_cell = cols[0].text.strip()
|
||||
position_match = re.search(r'([\d\.]+)°([EW])', position_cell)
|
||||
position_match = re.search(r"([\d\.]+)°([EW])", position_cell)
|
||||
if position_match:
|
||||
position_value = position_match.group(1)
|
||||
position_direction = position_match.group(2)
|
||||
position = f"{position_value}{position_direction}"
|
||||
else:
|
||||
position = None
|
||||
|
||||
|
||||
# Название спутника (2-я колонка)
|
||||
satellite_cell = cols[1]
|
||||
satellite_name = satellite_cell.get_text(strip=True)
|
||||
# Удаляем возможные лишние символы или пробелы
|
||||
satellite_name = re.sub(r'\s+', ' ', satellite_name).strip()
|
||||
|
||||
satellite_name = re.sub(r"\s+", " ", satellite_name).strip()
|
||||
|
||||
# NORAD (3-я колонка)
|
||||
norad = cols[2].text.strip()
|
||||
if not norad or norad == "-":
|
||||
norad = None
|
||||
|
||||
|
||||
ini_link = None
|
||||
ini_cell = cols[3]
|
||||
ini_img = ini_cell.find('img', src=lambda x: x and 'disquette.gif' in x)
|
||||
ini_img = ini_cell.find("img", src=lambda x: x and "disquette.gif" in x)
|
||||
if ini_img and position:
|
||||
ini_link = f"https://ru.kingofsat.net/dl.php?pos={position}&fkhz=0"
|
||||
|
||||
|
||||
update_date = cols[12].text.strip() if len(cols) > 12 else None
|
||||
|
||||
|
||||
if satellite_name and ini_link and position:
|
||||
satellites.append({
|
||||
'position': position,
|
||||
'name': satellite_name,
|
||||
'norad': norad,
|
||||
'ini_url': ini_link,
|
||||
'update_date': update_date
|
||||
})
|
||||
|
||||
satellites.append(
|
||||
{
|
||||
"position": position,
|
||||
"name": satellite_name,
|
||||
"norad": norad,
|
||||
"ini_url": ini_link,
|
||||
"update_date": update_date,
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Ошибка при обработке строки таблицы: {e}")
|
||||
continue
|
||||
|
||||
|
||||
return satellites
|
||||
|
||||
|
||||
def parse_ini_file(self, ini_content):
|
||||
"""Распарсить содержимое .ini файла"""
|
||||
data = {
|
||||
'metadata': {},
|
||||
'sattype': {},
|
||||
'dvb': {}
|
||||
}
|
||||
|
||||
data = {"metadata": {}, "sattype": {}, "dvb": {}}
|
||||
|
||||
# # Извлекаем метаданные из комментариев
|
||||
# metadata_match = re.search(r'\[ downloaded from www\.kingofsat\.net \(c\) (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \]', ini_content)
|
||||
# if metadata_match:
|
||||
# data['metadata']['downloaded'] = metadata_match.group(1)
|
||||
|
||||
|
||||
# Парсим секцию [SATTYPE]
|
||||
sattype_match = re.search(r'\[SATTYPE\](.*?)\n\[', ini_content, re.DOTALL)
|
||||
sattype_match = re.search(r"\[SATTYPE\](.*?)\n\[", ini_content, re.DOTALL)
|
||||
if sattype_match:
|
||||
sattype_content = sattype_match.group(1).strip()
|
||||
for line in sattype_content.split('\n'):
|
||||
for line in sattype_content.split("\n"):
|
||||
line = line.strip()
|
||||
if '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
data['sattype'][key.strip()] = value.strip()
|
||||
|
||||
if "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
data["sattype"][key.strip()] = value.strip()
|
||||
|
||||
# Парсим секцию [DVB]
|
||||
dvb_match = re.search(r'\[DVB\](.*?)(?:\n\[|$)', ini_content, re.DOTALL)
|
||||
dvb_match = re.search(r"\[DVB\](.*?)(?:\n\[|$)", ini_content, re.DOTALL)
|
||||
if dvb_match:
|
||||
dvb_content = dvb_match.group(1).strip()
|
||||
for line in dvb_content.split('\n'):
|
||||
for line in dvb_content.split("\n"):
|
||||
line = line.strip()
|
||||
if '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
params = [p.strip() for p in value.split(',')]
|
||||
polarization = params[1] if len(params) > 1 else ''
|
||||
if "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
params = [p.strip() for p in value.split(",")]
|
||||
polarization = params[1] if len(params) > 1 else ""
|
||||
if polarization:
|
||||
polarization = self.convert_polarization(polarization)
|
||||
|
||||
data['dvb'][key.strip()] = {
|
||||
'frequency': params[0] if len(params) > 0 else '',
|
||||
'polarization': polarization,
|
||||
'symbol_rate': params[2] if len(params) > 2 else '',
|
||||
'fec': params[3] if len(params) > 3 else '',
|
||||
'standard': params[4] if len(params) > 4 else '',
|
||||
'modulation': params[5] if len(params) > 5 else ''
|
||||
|
||||
data["dvb"][key.strip()] = {
|
||||
"frequency": params[0] if len(params) > 0 else "",
|
||||
"polarization": polarization,
|
||||
"symbol_rate": params[2] if len(params) > 2 else "",
|
||||
"fec": params[3] if len(params) > 3 else "",
|
||||
"standard": params[4] if len(params) > 4 else "",
|
||||
"modulation": params[5] if len(params) > 5 else "",
|
||||
}
|
||||
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def download_ini_file(self, url):
|
||||
"""Скачать содержимое .ini файла"""
|
||||
try:
|
||||
@@ -333,71 +340,66 @@ class KingOfSatParser:
|
||||
except Exception as e:
|
||||
print(f"Ошибка при скачивании .ini файла {url}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_all_satellites_data(self):
|
||||
"""Получить данные всех спутников с учетом ограничения max_satellites"""
|
||||
html_content = self.fetch_page(self.base_url + '/satellites')
|
||||
html_content = self.fetch_page(self.base_url + "/satellites")
|
||||
if not html_content:
|
||||
return []
|
||||
|
||||
|
||||
satellites = self.parse_satellite_table(html_content)
|
||||
|
||||
|
||||
if self.max_satellites > 0 and len(satellites) > self.max_satellites:
|
||||
satellites = satellites[:self.max_satellites]
|
||||
|
||||
satellites = satellites[: self.max_satellites]
|
||||
|
||||
results = []
|
||||
processed_count = 0
|
||||
|
||||
|
||||
for satellite in satellites:
|
||||
print(f"Обработка спутника: {satellite['name']} ({satellite['position']})")
|
||||
|
||||
ini_content = self.download_ini_file(satellite['ini_url'])
|
||||
|
||||
ini_content = self.download_ini_file(satellite["ini_url"])
|
||||
if not ini_content:
|
||||
print(f"Не удалось скачать .ini файл для {satellite['name']}")
|
||||
continue
|
||||
|
||||
|
||||
parsed_ini = self.parse_ini_file(ini_content)
|
||||
|
||||
|
||||
result = {
|
||||
'satellite_name': satellite['name'],
|
||||
'position': satellite['position'],
|
||||
'norad': satellite['norad'],
|
||||
'update_date': satellite['update_date'],
|
||||
'ini_url': satellite['ini_url'],
|
||||
'ini_data': parsed_ini
|
||||
"satellite_name": satellite["name"],
|
||||
"position": satellite["position"],
|
||||
"norad": satellite["norad"],
|
||||
"update_date": satellite["update_date"],
|
||||
"ini_url": satellite["ini_url"],
|
||||
"ini_data": parsed_ini,
|
||||
}
|
||||
|
||||
|
||||
results.append(result)
|
||||
processed_count += 1
|
||||
|
||||
|
||||
if self.max_satellites > 0 and processed_count >= self.max_satellites:
|
||||
break
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def create_satellite_dict(self, satellites_data):
|
||||
"""Создать словарь с данными спутников"""
|
||||
satellite_dict = {}
|
||||
|
||||
|
||||
for data in satellites_data:
|
||||
key = f"{data['position']}_{data['satellite_name'].replace(' ', '_').replace('/', '_')}"
|
||||
satellite_dict[key] = {
|
||||
'name': data['satellite_name'],
|
||||
'position': data['position'],
|
||||
'norad': data['norad'],
|
||||
'update_date': data['update_date'],
|
||||
'ini_url': data['ini_url'],
|
||||
'transponders_count': len(data['ini_data']['dvb']),
|
||||
'transponders': data['ini_data']['dvb'],
|
||||
'sattype_info': data['ini_data']['sattype'],
|
||||
'metadata': data['ini_data']['metadata']
|
||||
"name": data["satellite_name"],
|
||||
"position": data["position"],
|
||||
"norad": data["norad"],
|
||||
"update_date": data["update_date"],
|
||||
"ini_url": data["ini_url"],
|
||||
"transponders_count": len(data["ini_data"]["dvb"]),
|
||||
"transponders": data["ini_data"]["dvb"],
|
||||
"sattype_info": data["ini_data"]["sattype"],
|
||||
"metadata": data["ini_data"]["metadata"],
|
||||
}
|
||||
|
||||
return satellite_dict
|
||||
|
||||
from pprint import pprint
|
||||
lyngsat = LyngSatParser(regions=['europe'], target_sats=['Türksat 3A', 'Intelsat 22'])
|
||||
html_regions = lyngsat.get_region_pages()
|
||||
pprint(lyngsat.get_satellite_urls(html_regions))
|
||||
return satellite_dict
|
||||
|
||||
73
dbapp/lyngsatapp/tasks.py
Normal file
73
dbapp/lyngsatapp/tasks.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
Celery tasks for Lyngsat data processing.
|
||||
"""
|
||||
import logging
|
||||
from celery import shared_task
|
||||
from django.core.cache import cache
|
||||
|
||||
from .utils import fill_lyngsat_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task(bind=True, name='lyngsatapp.fill_lyngsat_data_async')
|
||||
def fill_lyngsat_data_task(self, target_sats, regions=None):
|
||||
"""
|
||||
Асинхронная задача для заполнения данных Lyngsat.
|
||||
|
||||
Args:
|
||||
target_sats: Список названий спутников для обработки
|
||||
regions: Список регионов для парсинга (по умолчанию все)
|
||||
|
||||
Returns:
|
||||
dict: Статистика обработки
|
||||
"""
|
||||
task_id = self.request.id
|
||||
logger.info(f"[Task {task_id}] Начало обработки данных Lyngsat")
|
||||
logger.info(f"[Task {task_id}] Спутники: {', '.join(target_sats)}")
|
||||
logger.info(f"[Task {task_id}] Регионы: {', '.join(regions) if regions else 'все'}")
|
||||
|
||||
# Обновляем статус задачи
|
||||
self.update_state(
|
||||
state='PROGRESS',
|
||||
meta={
|
||||
'current': 0,
|
||||
'total': len(target_sats),
|
||||
'status': 'Инициализация...'
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
# Вызываем функцию заполнения данных
|
||||
stats = fill_lyngsat_data(
|
||||
target_sats=target_sats,
|
||||
regions=regions,
|
||||
task_id=task_id,
|
||||
update_progress=lambda current, total, status: self.update_state(
|
||||
state='PROGRESS',
|
||||
meta={
|
||||
'current': current,
|
||||
'total': total,
|
||||
'status': status
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"[Task {task_id}] Обработка завершена успешно")
|
||||
logger.info(f"[Task {task_id}] Статистика: {stats}")
|
||||
|
||||
# Сохраняем результат в кеш для отображения на странице
|
||||
cache.set(f'lyngsat_task_{task_id}', stats, timeout=3600)
|
||||
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[Task {task_id}] Ошибка при обработке: {str(e)}", exc_info=True)
|
||||
self.update_state(
|
||||
state='FAILURE',
|
||||
meta={
|
||||
'error': str(e),
|
||||
'status': 'Ошибка при обработке'
|
||||
}
|
||||
)
|
||||
raise
|
||||
@@ -1,58 +1,170 @@
|
||||
import logging
|
||||
from .parser import LyngSatParser
|
||||
from .models import LyngSat
|
||||
from mainapp.models import Polarization, Standard, Modulation, Satellite
|
||||
|
||||
def fill_lyngsat_data(target_sats: list[str]):
|
||||
parser = LyngSatParser(
|
||||
target_sats=target_sats,
|
||||
)
|
||||
lyngsat_data = parser.get_satellites_data()
|
||||
for sat_name, data in lyngsat_data.items():
|
||||
url = data['url']
|
||||
sources = data['sources']
|
||||
for source in sources:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fill_lyngsat_data(
|
||||
target_sats: list[str],
|
||||
regions: list[str] = None,
|
||||
task_id: str = None,
|
||||
update_progress=None
|
||||
):
|
||||
"""
|
||||
Заполняет данные Lyngsat для указанных спутников и регионов.
|
||||
|
||||
Args:
|
||||
target_sats: Список названий спутников для обработки
|
||||
regions: Список регионов для парсинга (по умолчанию все)
|
||||
task_id: ID задачи Celery для логирования
|
||||
update_progress: Функция для обновления прогресса (current, total, status)
|
||||
|
||||
Returns:
|
||||
dict: Статистика обработки с ключами:
|
||||
- total_satellites: общее количество спутников
|
||||
- total_sources: общее количество источников
|
||||
- created: количество созданных записей
|
||||
- updated: количество обновленных записей
|
||||
- errors: список ошибок
|
||||
"""
|
||||
log_prefix = f"[Task {task_id}]" if task_id else "[Lyngsat]"
|
||||
stats = {
|
||||
'total_satellites': 0,
|
||||
'total_sources': 0,
|
||||
'created': 0,
|
||||
'updated': 0,
|
||||
'errors': []
|
||||
}
|
||||
|
||||
if regions is None:
|
||||
regions = ["europe", "asia", "america", "atlantic"]
|
||||
|
||||
logger.info(f"{log_prefix} Начало парсинга данных")
|
||||
logger.info(f"{log_prefix} Спутники: {', '.join(target_sats)}")
|
||||
logger.info(f"{log_prefix} Регионы: {', '.join(regions)}")
|
||||
|
||||
if update_progress:
|
||||
update_progress(0, len(target_sats), "Инициализация парсера...")
|
||||
|
||||
try:
|
||||
parser = LyngSatParser(
|
||||
target_sats=target_sats,
|
||||
regions=regions
|
||||
)
|
||||
|
||||
logger.info(f"{log_prefix} Получение данных со спутников...")
|
||||
if update_progress:
|
||||
update_progress(0, len(target_sats), "Получение данных со спутников...")
|
||||
|
||||
lyngsat_data = parser.get_satellites_data()
|
||||
stats['total_satellites'] = len(lyngsat_data)
|
||||
|
||||
logger.info(f"{log_prefix} Получено данных по {stats['total_satellites']} спутникам")
|
||||
|
||||
for idx, (sat_name, data) in enumerate(lyngsat_data.items(), 1):
|
||||
logger.info(f"{log_prefix} Обработка спутника {idx}/{stats['total_satellites']}: {sat_name}")
|
||||
|
||||
if update_progress:
|
||||
update_progress(idx, stats['total_satellites'], f"Обработка {sat_name}...")
|
||||
|
||||
url = data['url']
|
||||
sources = data['sources']
|
||||
stats['total_sources'] += len(sources)
|
||||
|
||||
logger.info(f"{log_prefix} Найдено {len(sources)} источников для {sat_name}")
|
||||
|
||||
# Находим спутник в базе
|
||||
try:
|
||||
freq = float(source['freq'])
|
||||
except Exception as e:
|
||||
freq = -1.0
|
||||
print("Беда с частотой")
|
||||
last_update = source['last_update']
|
||||
fec = source['metadata']['fec']
|
||||
modulation = source['metadata']['modulation']
|
||||
standard = source['metadata']['standard']
|
||||
symbol_velocity = source['metadata']['symbol_rate']
|
||||
polarization = source['pol']
|
||||
channel_info = source['provider_name']
|
||||
sat_obj = Satellite.objects.get(name__icontains=sat_name)
|
||||
logger.debug(f"{log_prefix} Спутник {sat_name} найден в базе (ID: {sat_obj.id})")
|
||||
except Satellite.DoesNotExist:
|
||||
error_msg = f"Спутник '{sat_name}' не найден в базе данных"
|
||||
logger.warning(f"{log_prefix} {error_msg}")
|
||||
stats['errors'].append(error_msg)
|
||||
continue
|
||||
except Satellite.MultipleObjectsReturned:
|
||||
error_msg = f"Найдено несколько спутников с именем '{sat_name}'"
|
||||
logger.warning(f"{log_prefix} {error_msg}")
|
||||
stats['errors'].append(error_msg)
|
||||
continue
|
||||
|
||||
for source_idx, source in enumerate(sources, 1):
|
||||
try:
|
||||
# Парсим частоту
|
||||
try:
|
||||
freq = float(source['freq'])
|
||||
except (ValueError, TypeError):
|
||||
freq = -1.0
|
||||
error_msg = f"Некорректная частота для {sat_name}: {source.get('freq')}"
|
||||
logger.debug(f"{log_prefix} {error_msg}")
|
||||
stats['errors'].append(error_msg)
|
||||
|
||||
last_update = source['last_update']
|
||||
fec = source['metadata'].get('fec')
|
||||
modulation_name = source['metadata'].get('modulation')
|
||||
standard_name = source['metadata'].get('standard')
|
||||
symbol_velocity = source['metadata'].get('symbol_rate')
|
||||
polarization_name = source['pol']
|
||||
channel_info = source['provider_name']
|
||||
|
||||
pol_obj, _ = Polarization.objects.get_or_create(
|
||||
name=polarization
|
||||
)
|
||||
# Создаем или получаем связанные объекты
|
||||
pol_obj, _ = Polarization.objects.get_or_create(
|
||||
name=polarization_name if polarization_name else "-"
|
||||
)
|
||||
|
||||
mod_obj, _ = Modulation.objects.get_or_create(
|
||||
name=modulation
|
||||
)
|
||||
mod_obj, _ = Modulation.objects.get_or_create(
|
||||
name=modulation_name if modulation_name else "-"
|
||||
)
|
||||
|
||||
standard_obj, _ = Standard.objects.get_or_create(
|
||||
name=standard
|
||||
)
|
||||
standard_obj, _ = Standard.objects.get_or_create(
|
||||
name=standard_name if standard_name else "-"
|
||||
)
|
||||
|
||||
sat_obj, _ = Satellite.objects.get(
|
||||
name__contains=sat_name
|
||||
)
|
||||
lyng_obj, _ = LyngSat.objects.get_or_create(
|
||||
id_satellite=sat_obj,
|
||||
frequency=freq,
|
||||
polarization=pol_obj,
|
||||
defaults={
|
||||
"modulation": mod_obj,
|
||||
"standard": standard_obj,
|
||||
"sym_velocity": symbol_velocity,
|
||||
"channel_info": channel_info,
|
||||
"last_update": last_update,
|
||||
"fec": fec,
|
||||
"url": url
|
||||
}
|
||||
)
|
||||
lyng_obj.objects.update_or_create()
|
||||
# TODO: сделать карточку и форму для действий и выбора спутника
|
||||
lyng_obj.save()
|
||||
# Создаем или обновляем запись Lyngsat
|
||||
lyng_obj, created = LyngSat.objects.update_or_create(
|
||||
id_satellite=sat_obj,
|
||||
frequency=freq,
|
||||
polarization=pol_obj,
|
||||
defaults={
|
||||
"modulation": mod_obj,
|
||||
"standard": standard_obj,
|
||||
"sym_velocity": symbol_velocity if symbol_velocity else 0,
|
||||
"channel_info": channel_info[:20] if channel_info else "",
|
||||
"last_update": last_update,
|
||||
"fec": fec[:30] if fec else "",
|
||||
"url": url
|
||||
}
|
||||
)
|
||||
|
||||
if created:
|
||||
stats['created'] += 1
|
||||
logger.debug(f"{log_prefix} Создана запись для {sat_name} {freq} МГц")
|
||||
else:
|
||||
stats['updated'] += 1
|
||||
logger.debug(f"{log_prefix} Обновлена запись для {sat_name} {freq} МГц")
|
||||
|
||||
# Логируем прогресс каждые 10 источников
|
||||
if source_idx % 10 == 0:
|
||||
logger.info(f"{log_prefix} Обработано {source_idx}/{len(sources)} источников для {sat_name}")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Ошибка при обработке источника {sat_name}: {str(e)}"
|
||||
logger.error(f"{log_prefix} {error_msg}", exc_info=True)
|
||||
stats['errors'].append(error_msg)
|
||||
continue
|
||||
|
||||
logger.info(f"{log_prefix} Завершена обработка {sat_name}: создано {stats['created']}, обновлено {stats['updated']}")
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Критическая ошибка: {str(e)}"
|
||||
logger.error(f"{log_prefix} {error_msg}", exc_info=True)
|
||||
stats['errors'].append(error_msg)
|
||||
|
||||
logger.info(f"{log_prefix} Обработка завершена. Итого: создано {stats['created']}, обновлено {stats['updated']}, ошибок {len(stats['errors'])}")
|
||||
|
||||
if update_progress:
|
||||
update_progress(stats['total_satellites'], stats['total_satellites'], "Завершено")
|
||||
|
||||
return stats
|
||||
|
||||
@@ -107,6 +107,40 @@ class NewEventForm(forms.Form):
|
||||
'accept': '.xlsx,.xls'
|
||||
})
|
||||
)
|
||||
|
||||
|
||||
class FillLyngsatDataForm(forms.Form):
|
||||
"""Форма для заполнения данных из Lyngsat"""
|
||||
|
||||
REGION_CHOICES = [
|
||||
('europe', 'Европа'),
|
||||
('asia', 'Азия'),
|
||||
('america', 'Америка'),
|
||||
('atlantic', 'Атлантика'),
|
||||
]
|
||||
|
||||
satellites = forms.ModelMultipleChoiceField(
|
||||
queryset=Satellite.objects.all().order_by('name'),
|
||||
label="Выберите спутники",
|
||||
widget=forms.SelectMultiple(attrs={
|
||||
'class': 'form-select',
|
||||
'size': '10'
|
||||
}),
|
||||
required=True,
|
||||
help_text="Удерживайте Ctrl (Cmd на Mac) для выбора нескольких спутников"
|
||||
)
|
||||
|
||||
regions = forms.MultipleChoiceField(
|
||||
choices=REGION_CHOICES,
|
||||
label="Выберите регионы",
|
||||
widget=forms.SelectMultiple(attrs={
|
||||
'class': 'form-select',
|
||||
'size': '4'
|
||||
}),
|
||||
required=True,
|
||||
initial=['europe', 'asia', 'america', 'atlantic'],
|
||||
help_text="Удерживайте Ctrl (Cmd на Mac) для выбора нескольких регионов"
|
||||
)
|
||||
class ParameterForm(forms.ModelForm):
|
||||
"""
|
||||
Форма для создания и редактирования параметров ВЧ загрузки.
|
||||
|
||||
@@ -124,23 +124,23 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Map Views Card -->
|
||||
<!-- Lyngsat Data Fill Card -->
|
||||
<div class="col-lg-6">
|
||||
<div class="card h-100 shadow-sm border-0">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-center mb-3">
|
||||
<div class="bg-secondary bg-opacity-10 rounded-circle p-2 me-3">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-map text-secondary" viewBox="0 0 16 16">
|
||||
<path d="M15.817.113A.5.5 0 0 1 16 .5v14a.5.5 0 0 1-.402.49l-5 1a.502.502 0 0 1-.196 0L5.5 15.01l-4.902.98A.5.5 0 0 1 0 15.5v-14a.5.5 0 0 1 .402-.49l5-1a.5.5 0 0 1 .196 0L10.5.99l4.902-.98a.5.5 0 0 1 .415.103M10 1.91l-4-.8v12.98l4 .8zM1.61 2.22l4.39.88v10.88l-4.39-.88zm9.18 10.88 4-.8V2.34l-4 .8z"/>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-cloud-download text-secondary" viewBox="0 0 16 16">
|
||||
<path d="M4.406 1.342A5.53 5.53 0 0 1 8 0c2.69 0 4.923 2 5.166 4.579C14.758 4.804 16 6.137 16 7.773 16 9.569 14.502 11 12.687 11H10a.5.5 0 0 1 0-1h2.688C13.979 10 15 8.988 15 7.773c0-1.216-1.02-2.228-2.313-2.228h-.5v-.5C12.188 2.825 10.328 1 8 1a4.53 4.53 0 0 0-2.941 1.1c-.757.652-1.153 1.438-1.153 2.055v.448l-.445.049C2.064 4.805 1 5.952 1 7.318 1 8.785 2.23 10 3.781 10H6a.5.5 0 0 1 0 1H3.781C1.708 11 0 9.366 0 7.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383"/>
|
||||
<path d="M7.646 15.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 14.293V5.5a.5.5 0 0 0-1 0v8.793l-2.146-2.147a.5.5 0 0 0-.708.708z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<h3 class="card-title mb-0">Карты</h3>
|
||||
</div>
|
||||
<p class="card-text">Просматривайте данные на 2D и 3D картах для визуализации геолокации спутников.</p>
|
||||
<div class="mt-2">
|
||||
<a href="{% url 'mapsapp:2dmap' %}" class="btn btn-secondary me-2">2D Карта</a>
|
||||
<a href="{% url 'mapsapp:3dmap' %}" class="btn btn-outline-secondary">3D Карта</a>
|
||||
<h3 class="card-title mb-0">Заполнение данных Lyngsat</h3>
|
||||
</div>
|
||||
<p class="card-text">Загрузите данные о транспондерах спутников с сайта Lyngsat. Выберите спутники и регионы для парсинга данных.</p>
|
||||
<a href="{% url 'mainapp:fill_lyngsat_data' %}" class="btn btn-secondary">
|
||||
Заполнить данные Lyngsat
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
118
dbapp/mainapp/templates/mainapp/fill_lyngsat_data.html
Normal file
118
dbapp/mainapp/templates/mainapp/fill_lyngsat_data.html
Normal file
@@ -0,0 +1,118 @@
|
||||
{% extends 'mainapp/base.html' %}
|
||||
|
||||
{% block title %}Заполнение данных Lyngsat{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container mt-4">
|
||||
<div class="row justify-content-center">
|
||||
<div class="col-lg-8">
|
||||
<div class="card shadow-sm">
|
||||
<div class="card-header bg-primary text-white">
|
||||
<h3 class="mb-0">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-cloud-download me-2" viewBox="0 0 16 16">
|
||||
<path d="M4.406 1.342A5.53 5.53 0 0 1 8 0c2.69 0 4.923 2 5.166 4.579C14.758 4.804 16 6.137 16 7.773 16 9.569 14.502 11 12.687 11H10a.5.5 0 0 1 0-1h2.688C13.979 10 15 8.988 15 7.773c0-1.216-1.02-2.228-2.313-2.228h-.5v-.5C12.188 2.825 10.328 1 8 1a4.53 4.53 0 0 0-2.941 1.1c-.757.652-1.153 1.438-1.153 2.055v.448l-.445.049C2.064 4.805 1 5.952 1 7.318 1 8.785 2.23 10 3.781 10H6a.5.5 0 0 1 0 1H3.781C1.708 11 0 9.366 0 7.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383"/>
|
||||
<path d="M7.646 15.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 14.293V5.5a.5.5 0 0 0-1 0v8.793l-2.146-2.147a.5.5 0 0 0-.708.708z"/>
|
||||
</svg>
|
||||
Заполнение данных из Lyngsat
|
||||
</h3>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<!-- Alert messages -->
|
||||
{% include 'mainapp/components/_messages.html' %}
|
||||
|
||||
<div class="alert alert-info" role="alert">
|
||||
<strong>Внимание!</strong> Процесс заполнения данных может занять продолжительное время,
|
||||
так как выполняются запросы к внешнему сайту Lyngsat. Пожалуйста, дождитесь завершения операции.
|
||||
</div>
|
||||
|
||||
<form method="post" class="needs-validation" novalidate>
|
||||
{% csrf_token %}
|
||||
|
||||
<!-- Satellites Selection -->
|
||||
<div class="mb-4">
|
||||
<label for="{{ form.satellites.id_for_label }}" class="form-label fw-bold">
|
||||
{{ form.satellites.label }}
|
||||
</label>
|
||||
{{ form.satellites }}
|
||||
{% if form.satellites.help_text %}
|
||||
<div class="form-text">{{ form.satellites.help_text }}</div>
|
||||
{% endif %}
|
||||
{% if form.satellites.errors %}
|
||||
<div class="invalid-feedback d-block">
|
||||
{{ form.satellites.errors }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Regions Selection -->
|
||||
<div class="mb-4">
|
||||
<label for="{{ form.regions.id_for_label }}" class="form-label fw-bold">
|
||||
{{ form.regions.label }}
|
||||
</label>
|
||||
{{ form.regions }}
|
||||
{% if form.regions.help_text %}
|
||||
<div class="form-text">{{ form.regions.help_text }}</div>
|
||||
{% endif %}
|
||||
{% if form.regions.errors %}
|
||||
<div class="invalid-feedback d-block">
|
||||
{{ form.regions.errors }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- Buttons -->
|
||||
<div class="d-grid gap-2 d-md-flex justify-content-md-between">
|
||||
<a href="{% url 'mainapp:actions' %}" class="btn btn-secondary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-arrow-left me-1" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M15 8a.5.5 0 0 0-.5-.5H2.707l3.147-3.146a.5.5 0 1 0-.708-.708l-4 4a.5.5 0 0 0 0 .708l4 4a.5.5 0 0 0 .708-.708L2.707 8.5H14.5A.5.5 0 0 0 15 8"/>
|
||||
</svg>
|
||||
Назад
|
||||
</a>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-download me-1" viewBox="0 0 16 16">
|
||||
<path d="M.5 9.9a.5.5 0 0 1 .5.5v2.5a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1v-2.5a.5.5 0 0 1 1 0v2.5a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2v-2.5a.5.5 0 0 1 .5-.5"/>
|
||||
<path d="M7.646 11.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 10.293V1.5a.5.5 0 0 0-1 0v8.793L5.354 8.146a.5.5 0 1 0-.708.708z"/>
|
||||
</svg>
|
||||
Заполнить данные
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Info Card -->
|
||||
<div class="card mt-4 shadow-sm">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">Информация</h5>
|
||||
<p class="card-text">
|
||||
Эта форма позволяет загрузить данные о транспондерах спутников с сайта Lyngsat.
|
||||
Выберите один или несколько спутников и регионы для парсинга данных.
|
||||
</p>
|
||||
<ul>
|
||||
<li>Данные включают частоты, поляризацию, модуляцию, стандарты и другие параметры</li>
|
||||
<li>Процесс может занять несколько минут в зависимости от количества выбранных спутников</li>
|
||||
<li>Существующие записи будут обновлены, новые - созданы</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Form validation
|
||||
(function() {
|
||||
'use strict';
|
||||
var forms = document.querySelectorAll('.needs-validation');
|
||||
Array.prototype.slice.call(forms).forEach(function(form) {
|
||||
form.addEventListener('submit', function(event) {
|
||||
if (!form.checkValidity()) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}
|
||||
form.classList.add('was-validated');
|
||||
}, false);
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
241
dbapp/mainapp/templates/mainapp/lyngsat_task_status.html
Normal file
241
dbapp/mainapp/templates/mainapp/lyngsat_task_status.html
Normal file
@@ -0,0 +1,241 @@
|
||||
{% extends 'mainapp/base.html' %}
|
||||
|
||||
{% block title %}Статус задачи Lyngsat{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container mt-4">
|
||||
<div class="row justify-content-center">
|
||||
<div class="col-lg-8">
|
||||
<div class="card shadow-sm">
|
||||
<div class="card-header bg-primary text-white">
|
||||
<h3 class="mb-0">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-hourglass-split me-2" viewBox="0 0 16 16">
|
||||
<path d="M2.5 15a.5.5 0 1 1 0-1h1v-1a4.5 4.5 0 0 1 2.557-4.06c.29-.139.443-.377.443-.59v-.7c0-.213-.154-.451-.443-.59A4.5 4.5 0 0 1 3.5 3V2h-1a.5.5 0 0 1 0-1h11a.5.5 0 0 1 0 1h-1v1a4.5 4.5 0 0 1-2.557 4.06c-.29.139-.443.377-.443.59v.7c0 .213.154.451.443.59A4.5 4.5 0 0 1 12.5 13v1h1a.5.5 0 0 1 0 1zm2-13v1c0 .537.12 1.045.337 1.5h6.326c.216-.455.337-.963.337-1.5V2zm3 6.35c0 .701-.478 1.236-1.011 1.492A3.5 3.5 0 0 0 4.5 13s.866-1.299 3-1.48zm1 0v3.17c2.134.181 3 1.48 3 1.48a3.5 3.5 0 0 0-1.989-3.158C8.978 9.586 8.5 9.052 8.5 8.351z"/>
|
||||
</svg>
|
||||
Статус задачи заполнения данных Lyngsat
|
||||
</h3>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
{% if task_id %}
|
||||
<div class="mb-3">
|
||||
<strong>ID задачи:</strong> <code id="task-id">{{ task_id }}</code>
|
||||
</div>
|
||||
|
||||
<!-- Progress Bar -->
|
||||
<div class="mb-4">
|
||||
<div class="d-flex justify-content-between mb-2">
|
||||
<span id="status-text">Загрузка статуса...</span>
|
||||
<span id="progress-percent">0%</span>
|
||||
</div>
|
||||
<div class="progress" style="height: 25px;">
|
||||
<div id="progress-bar" class="progress-bar progress-bar-striped progress-bar-animated"
|
||||
role="progressbar" style="width: 0%;"
|
||||
aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
|
||||
<span id="progress-text">0%</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Task State -->
|
||||
<div id="task-state-container" class="alert alert-info" role="alert">
|
||||
<strong>Состояние:</strong> <span id="task-state">Проверка...</span>
|
||||
</div>
|
||||
|
||||
<!-- Results Container (hidden by default) -->
|
||||
<div id="results-container" class="d-none">
|
||||
<h5 class="mt-4">Результаты обработки</h5>
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<div class="card mb-3">
|
||||
<div class="card-body">
|
||||
<h6 class="card-subtitle mb-2 text-muted">Обработано спутников</h6>
|
||||
<h3 class="card-title" id="result-satellites">-</h3>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mb-3">
|
||||
<div class="card-body">
|
||||
<h6 class="card-subtitle mb-2 text-muted">Обработано источников</h6>
|
||||
<h3 class="card-title" id="result-sources">-</h3>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mb-3 border-success">
|
||||
<div class="card-body">
|
||||
<h6 class="card-subtitle mb-2 text-success">Создано записей</h6>
|
||||
<h3 class="card-title text-success" id="result-created">-</h3>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mb-3 border-info">
|
||||
<div class="card-body">
|
||||
<h6 class="card-subtitle mb-2 text-info">Обновлено записей</h6>
|
||||
<h3 class="card-title text-info" id="result-updated">-</h3>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Errors -->
|
||||
<div id="errors-container" class="d-none">
|
||||
<h6 class="text-danger">Ошибки при обработке:</h6>
|
||||
<div class="alert alert-warning">
|
||||
<ul id="errors-list" class="mb-0"></ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Error Container (hidden by default) -->
|
||||
<div id="error-container" class="alert alert-danger d-none" role="alert">
|
||||
<strong>Ошибка:</strong> <span id="error-text"></span>
|
||||
</div>
|
||||
|
||||
<!-- Action Buttons -->
|
||||
<div class="d-grid gap-2 d-md-flex justify-content-md-between mt-4">
|
||||
<a href="{% url 'mainapp:fill_lyngsat_data' %}" class="btn btn-secondary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-arrow-left me-1" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M15 8a.5.5 0 0 0-.5-.5H2.707l3.147-3.146a.5.5 0 1 0-.708-.708l-4 4a.5.5 0 0 0 0 .708l4 4a.5.5 0 0 0 .708-.708L2.707 8.5H14.5A.5.5 0 0 0 15 8"/>
|
||||
</svg>
|
||||
Назад к форме
|
||||
</a>
|
||||
<a href="{% url 'mainapp:actions' %}" class="btn btn-outline-primary" id="actions-btn">
|
||||
Перейти к действиям
|
||||
</a>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="alert alert-warning" role="alert">
|
||||
ID задачи не указан. Пожалуйста, запустите задачу через форму заполнения данных.
|
||||
</div>
|
||||
<a href="{% url 'mainapp:fill_lyngsat_data' %}" class="btn btn-primary">
|
||||
Перейти к форме
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if task_id %}
|
||||
<script>
|
||||
let taskId = '{{ task_id }}';
|
||||
let pollInterval;
|
||||
let isCompleted = false;
|
||||
|
||||
function updateProgress(data) {
|
||||
const statusText = document.getElementById('status-text');
|
||||
const progressBar = document.getElementById('progress-bar');
|
||||
const progressText = document.getElementById('progress-text');
|
||||
const progressPercent = document.getElementById('progress-percent');
|
||||
const taskState = document.getElementById('task-state');
|
||||
const taskStateContainer = document.getElementById('task-state-container');
|
||||
|
||||
// Update state
|
||||
taskState.textContent = data.state;
|
||||
|
||||
if (data.state === 'PENDING') {
|
||||
statusText.textContent = 'Задача в очереди...';
|
||||
taskStateContainer.className = 'alert alert-info';
|
||||
} else if (data.state === 'PROGRESS') {
|
||||
const percent = data.percent || 0;
|
||||
statusText.textContent = data.status || 'Обработка...';
|
||||
progressBar.style.width = percent + '%';
|
||||
progressBar.setAttribute('aria-valuenow', percent);
|
||||
progressText.textContent = percent + '%';
|
||||
progressPercent.textContent = percent + '%';
|
||||
taskStateContainer.className = 'alert alert-info';
|
||||
} else if (data.state === 'SUCCESS') {
|
||||
statusText.textContent = 'Задача завершена успешно!';
|
||||
progressBar.style.width = '100%';
|
||||
progressBar.setAttribute('aria-valuenow', 100);
|
||||
progressText.textContent = '100%';
|
||||
progressPercent.textContent = '100%';
|
||||
progressBar.classList.remove('progress-bar-animated');
|
||||
progressBar.classList.add('bg-success');
|
||||
taskStateContainer.className = 'alert alert-success';
|
||||
|
||||
// Show results
|
||||
if (data.result) {
|
||||
showResults(data.result);
|
||||
}
|
||||
|
||||
isCompleted = true;
|
||||
clearInterval(pollInterval);
|
||||
} else if (data.state === 'FAILURE') {
|
||||
statusText.textContent = 'Ошибка при выполнении задачи';
|
||||
progressBar.classList.remove('progress-bar-animated');
|
||||
progressBar.classList.add('bg-danger');
|
||||
taskStateContainer.className = 'alert alert-danger';
|
||||
|
||||
// Show error
|
||||
const errorContainer = document.getElementById('error-container');
|
||||
const errorText = document.getElementById('error-text');
|
||||
errorText.textContent = data.error || 'Неизвестная ошибка';
|
||||
errorContainer.classList.remove('d-none');
|
||||
|
||||
isCompleted = true;
|
||||
clearInterval(pollInterval);
|
||||
}
|
||||
}
|
||||
|
||||
function showResults(result) {
|
||||
const resultsContainer = document.getElementById('results-container');
|
||||
resultsContainer.classList.remove('d-none');
|
||||
|
||||
document.getElementById('result-satellites').textContent = result.total_satellites || 0;
|
||||
document.getElementById('result-sources').textContent = result.total_sources || 0;
|
||||
document.getElementById('result-created').textContent = result.created || 0;
|
||||
document.getElementById('result-updated').textContent = result.updated || 0;
|
||||
|
||||
// Show errors if any
|
||||
if (result.errors && result.errors.length > 0) {
|
||||
const errorsContainer = document.getElementById('errors-container');
|
||||
const errorsList = document.getElementById('errors-list');
|
||||
errorsContainer.classList.remove('d-none');
|
||||
|
||||
errorsList.innerHTML = '';
|
||||
result.errors.slice(0, 10).forEach(error => {
|
||||
const li = document.createElement('li');
|
||||
li.textContent = error;
|
||||
errorsList.appendChild(li);
|
||||
});
|
||||
|
||||
if (result.errors.length > 10) {
|
||||
const li = document.createElement('li');
|
||||
li.textContent = `И еще ${result.errors.length - 10} ошибок...`;
|
||||
li.className = 'text-muted';
|
||||
errorsList.appendChild(li);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function checkTaskStatus() {
|
||||
fetch(`/api/lyngsat-task-status/${taskId}/`)
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
updateProgress(data);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error checking task status:', error);
|
||||
});
|
||||
}
|
||||
|
||||
// Start polling
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
checkTaskStatus();
|
||||
pollInterval = setInterval(checkTaskStatus, 2000); // Poll every 2 seconds
|
||||
|
||||
// Stop polling after 30 minutes
|
||||
setTimeout(() => {
|
||||
if (!isCompleted) {
|
||||
clearInterval(pollInterval);
|
||||
document.getElementById('status-text').textContent = 'Превышено время ожидания. Обновите страницу для проверки статуса.';
|
||||
}
|
||||
}, 30 * 60 * 1000);
|
||||
});
|
||||
</script>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
@@ -25,4 +25,8 @@ urlpatterns = [
|
||||
path('object/<int:pk>/edit/', views.ObjItemUpdateView.as_view(), name='objitem_update'),
|
||||
path('object/<int:pk>/', views.ObjItemDetailView.as_view(), name='objitem_detail'),
|
||||
path('object/<int:pk>/delete/', views.ObjItemDeleteView.as_view(), name='objitem_delete'),
|
||||
path('fill-lyngsat-data/', views.FillLyngsatDataView.as_view(), name='fill_lyngsat_data'),
|
||||
path('lyngsat-task-status/', views.LyngsatTaskStatusView.as_view(), name='lyngsat_task_status'),
|
||||
path('lyngsat-task-status/<str:task_id>/', views.LyngsatTaskStatusView.as_view(), name='lyngsat_task_status'),
|
||||
path('api/lyngsat-task-status/<str:task_id>/', views.LyngsatTaskStatusAPIView.as_view(), name='lyngsat_task_status_api'),
|
||||
]
|
||||
@@ -37,6 +37,7 @@ from .forms import (
|
||||
UploadFileForm,
|
||||
UploadVchLoad,
|
||||
VchLinkForm,
|
||||
FillLyngsatDataForm,
|
||||
)
|
||||
from .mixins import CoordinateProcessingMixin, FormMessageMixin, RoleRequiredMixin
|
||||
from .models import Geo, Modulation, ObjItem, Polarization, Satellite
|
||||
@@ -1029,3 +1030,97 @@ class ObjItemDetailView(LoginRequiredMixin, View):
|
||||
}
|
||||
|
||||
return render(request, "mainapp/objitem_detail.html", context)
|
||||
|
||||
|
||||
class FillLyngsatDataView(LoginRequiredMixin, FormMessageMixin, FormView):
|
||||
"""
|
||||
Представление для заполнения данных из Lyngsat.
|
||||
|
||||
Позволяет выбрать спутники и регионы для парсинга данных с сайта Lyngsat.
|
||||
Запускает асинхронную задачу Celery для обработки.
|
||||
"""
|
||||
template_name = "mainapp/fill_lyngsat_data.html"
|
||||
form_class = FillLyngsatDataForm
|
||||
success_url = reverse_lazy("mainapp:lyngsat_task_status")
|
||||
error_message = "Форма заполнена некорректно"
|
||||
|
||||
def form_valid(self, form):
|
||||
satellites = form.cleaned_data["satellites"]
|
||||
regions = form.cleaned_data["regions"]
|
||||
|
||||
# Получаем названия спутников
|
||||
target_sats = [sat.name for sat in satellites]
|
||||
|
||||
try:
|
||||
from lyngsatapp.tasks import fill_lyngsat_data_task
|
||||
|
||||
# Запускаем асинхронную задачу
|
||||
task = fill_lyngsat_data_task.delay(target_sats, regions)
|
||||
|
||||
messages.success(
|
||||
self.request,
|
||||
f"Задача запущена! ID задачи: {task.id}. "
|
||||
"Вы будете перенаправлены на страницу отслеживания прогресса."
|
||||
)
|
||||
|
||||
# Перенаправляем на страницу статуса задачи
|
||||
return redirect('mainapp:lyngsat_task_status', task_id=task.id)
|
||||
|
||||
except Exception as e:
|
||||
messages.error(self.request, f"Ошибка при запуске задачи: {str(e)}")
|
||||
return redirect("mainapp:fill_lyngsat_data")
|
||||
|
||||
|
||||
class LyngsatTaskStatusView(LoginRequiredMixin, View):
|
||||
"""
|
||||
Представление для отслеживания статуса задачи заполнения данных Lyngsat.
|
||||
"""
|
||||
template_name = "mainapp/lyngsat_task_status.html"
|
||||
|
||||
def get(self, request, task_id=None):
|
||||
context = {
|
||||
'task_id': task_id
|
||||
}
|
||||
return render(request, self.template_name, context)
|
||||
|
||||
|
||||
class LyngsatTaskStatusAPIView(LoginRequiredMixin, View):
|
||||
"""
|
||||
API для получения статуса задачи Celery.
|
||||
"""
|
||||
def get(self, request, task_id):
|
||||
from celery.result import AsyncResult
|
||||
from django.core.cache import cache
|
||||
|
||||
task = AsyncResult(task_id)
|
||||
|
||||
response_data = {
|
||||
'task_id': task_id,
|
||||
'state': task.state,
|
||||
'result': None,
|
||||
'error': None
|
||||
}
|
||||
|
||||
if task.state == 'PENDING':
|
||||
response_data['status'] = 'Задача в очереди...'
|
||||
elif task.state == 'PROGRESS':
|
||||
response_data['status'] = task.info.get('status', '')
|
||||
response_data['current'] = task.info.get('current', 0)
|
||||
response_data['total'] = task.info.get('total', 1)
|
||||
response_data['percent'] = int((task.info.get('current', 0) / task.info.get('total', 1)) * 100)
|
||||
elif task.state == 'SUCCESS':
|
||||
# Получаем результат из кеша
|
||||
result = cache.get(f'lyngsat_task_{task_id}')
|
||||
if result:
|
||||
response_data['result'] = result
|
||||
response_data['status'] = 'Задача завершена успешно'
|
||||
else:
|
||||
response_data['result'] = task.result
|
||||
response_data['status'] = 'Задача завершена'
|
||||
elif task.state == 'FAILURE':
|
||||
response_data['status'] = 'Ошибка при выполнении задачи'
|
||||
response_data['error'] = str(task.info)
|
||||
else:
|
||||
response_data['status'] = task.state
|
||||
|
||||
return JsonResponse(response_data)
|
||||
|
||||
@@ -24,6 +24,8 @@ pandas>=2.3.3
|
||||
psycopg>=3.2.10
|
||||
psycopg2-binary>=2.9.11
|
||||
redis>=6.4.0
|
||||
celery>=5.4.0
|
||||
django-celery-results>=2.5.1
|
||||
requests>=2.32.5
|
||||
reverse-geocoder>=1.5.1
|
||||
scikit-learn>=1.7.2
|
||||
|
||||
5
dbapp/start_celery_worker.sh
Executable file
5
dbapp/start_celery_worker.sh
Executable file
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
# Script to start Celery worker
|
||||
|
||||
echo "Starting Celery worker..."
|
||||
celery -A dbapp worker --loglevel=info --logfile=logs/celery_worker.log
|
||||
Reference in New Issue
Block a user