Добавил форму для загрузки данных с LyngSat

This commit is contained in:
2025-11-10 23:28:06 +03:00
parent 1b345a3fd9
commit 65e6c9a323
24 changed files with 2730 additions and 308 deletions

View File

@@ -0,0 +1,8 @@
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
try:
from .celery import app as celery_app
__all__ = ('celery_app',)
except ImportError:
# Celery is not installed, skip initialization
pass

24
dbapp/dbapp/celery.py Normal file
View File

@@ -0,0 +1,24 @@
"""
Celery configuration for dbapp project.
"""
import os
from celery import Celery
# Set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dbapp.settings.development')
app = Celery('dbapp')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django apps.
app.autodiscover_tasks()
@app.task(bind=True, ignore_result=True)
def debug_task(self):
print(f'Request: {self.request!r}')

View File

@@ -21,19 +21,21 @@ load_dotenv()
BASE_DIR = Path(__file__).resolve().parent.parent
# GDAL/GEOS configuration for Windows
if os.name == 'nt':
if os.name == "nt":
OSGEO4W = r"C:\Program Files\OSGeo4W"
assert os.path.isdir(OSGEO4W), "Directory does not exist: " + OSGEO4W
os.environ['OSGEO4W_ROOT'] = OSGEO4W
os.environ['PROJ_LIB'] = os.path.join(OSGEO4W, r"share\proj")
os.environ['PATH'] = OSGEO4W + r"\bin;" + os.environ['PATH']
os.environ["OSGEO4W_ROOT"] = OSGEO4W
os.environ["PROJ_LIB"] = os.path.join(OSGEO4W, r"share\proj")
os.environ["PATH"] = OSGEO4W + r"\bin;" + os.environ["PATH"]
# ============================================================================
# SECURITY SETTINGS
# ============================================================================
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.getenv('SECRET_KEY', 'django-insecure-7etj5f7buo2a57xv=w3^&llusq8rii7b_gd)9$t_1xcnao!^tq')
SECRET_KEY = os.getenv(
"SECRET_KEY", "django-insecure-7etj5f7buo2a57xv=w3^&llusq8rii7b_gd)9$t_1xcnao!^tq"
)
# SECURITY WARNING: don't run with debug turned on in production!
# This should be overridden in environment-specific settings
@@ -49,38 +51,42 @@ ALLOWED_HOSTS = []
INSTALLED_APPS = [
# Django Autocomplete Light (must be before admin)
'dal',
'dal_select2',
"dal",
"dal_select2",
# Admin interface customization
'admin_interface',
'colorfield',
"admin_interface",
"colorfield",
# Django GIS
'django.contrib.gis',
"django.contrib.gis",
# Django core apps
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.humanize',
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django.contrib.humanize",
# Third-party apps
'leaflet',
'dynamic_raw_id',
'rangefilter',
'django_admin_multiple_choice_list_filter',
'more_admin_filters',
'import_export',
"leaflet",
"dynamic_raw_id",
"rangefilter",
"django_admin_multiple_choice_list_filter",
"more_admin_filters",
"import_export",
# Project apps
'mainapp',
'mapsapp',
"mainapp",
"mapsapp",
"lyngsatapp",
]
# Add Celery results app if available
try:
import django_celery_results
INSTALLED_APPS.append("django_celery_results")
except ImportError:
pass
# Note: Custom user model is implemented via OneToOneField relationship
# If you need a custom user model, uncomment and configure:
# AUTH_USER_MODEL = 'mainapp.CustomUser'
@@ -90,17 +96,17 @@ INSTALLED_APPS = [
# ============================================================================
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.locale.LocaleMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = 'dbapp.urls'
ROOT_URLCONF = "dbapp.urls"
# ============================================================================
# TEMPLATES CONFIGURATION
@@ -108,36 +114,36 @@ ROOT_URLCONF = 'dbapp.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [
BASE_DIR / 'templates',
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [
BASE_DIR / "templates",
],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = 'dbapp.wsgi.application'
WSGI_APPLICATION = "dbapp.wsgi.application"
# ============================================================================
# DATABASE CONFIGURATION
# ============================================================================
DATABASES = {
'default': {
'ENGINE': os.getenv('DB_ENGINE', 'django.contrib.gis.db.backends.postgis'),
'NAME': os.getenv('DB_NAME', 'db'),
'USER': os.getenv('DB_USER', 'user'),
'PASSWORD': os.getenv('DB_PASSWORD', 'password'),
'HOST': os.getenv('DB_HOST', 'localhost'),
'PORT': os.getenv('DB_PORT', '5432'),
"default": {
"ENGINE": os.getenv("DB_ENGINE", "django.contrib.gis.db.backends.postgis"),
"NAME": os.getenv("DB_NAME", "db"),
"USER": os.getenv("DB_USER", "user"),
"PASSWORD": os.getenv("DB_PASSWORD", "password"),
"HOST": os.getenv("DB_HOST", "localhost"),
"PORT": os.getenv("DB_PORT", "5432"),
}
}
@@ -148,16 +154,16 @@ DATABASES = {
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
@@ -165,9 +171,9 @@ AUTH_PASSWORD_VALIDATORS = [
# INTERNATIONALIZATION
# ============================================================================
LANGUAGE_CODE = 'ru'
LANGUAGE_CODE = "ru"
TIME_ZONE = 'Europe/Moscow'
TIME_ZONE = "Europe/Moscow"
USE_I18N = True
@@ -177,18 +183,18 @@ USE_TZ = True
# AUTHENTICATION CONFIGURATION
# ============================================================================
LOGIN_URL = 'login'
LOGIN_REDIRECT_URL = 'mainapp:home'
LOGOUT_REDIRECT_URL = 'mainapp:home'
LOGIN_URL = "login"
LOGIN_REDIRECT_URL = "mainapp:home"
LOGOUT_REDIRECT_URL = "mainapp:home"
# ============================================================================
# STATIC FILES CONFIGURATION
# ============================================================================
STATIC_URL = '/static/'
STATIC_URL = "/static/"
STATICFILES_DIRS = [
BASE_DIR.parent / 'static',
BASE_DIR.parent / "static",
]
# STATIC_ROOT will be set in production.py
@@ -198,7 +204,7 @@ STATICFILES_DIRS = [
# ============================================================================
# Default primary key field type
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
# ============================================================================
# THIRD-PARTY APP CONFIGURATION
@@ -210,17 +216,53 @@ SILENCED_SYSTEM_CHECKS = ["security.W019"]
# Leaflet Configuration
LEAFLET_CONFIG = {
'ATTRIBUTION_PREFIX': '',
'TILES': [
"ATTRIBUTION_PREFIX": "",
"TILES": [
(
'Satellite',
'https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}',
{'attribution': '© Esri', 'maxZoom': 16}
"Satellite",
"https://server.arcgisonline.com/ArcGIS/rest/services/World_Imagery/MapServer/tile/{z}/{y}/{x}",
{"attribution": "© Esri", "maxZoom": 16},
),
(
'Streets',
'http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',
{'attribution': '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'}
)
"Streets",
"http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",
{
"attribution": '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a>'
},
),
],
}
}
# ============================================================================
# CELERY CONFIGURATION
# ============================================================================
# Celery Configuration Options
CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
CELERY_RESULT_BACKEND = "django-db"
CELERY_CACHE_BACKEND = "default"
# Celery Task Configuration
CELERY_TASK_TRACK_STARTED = True
CELERY_TASK_TIME_LIMIT = 30 * 60 # 30 minutes
CELERY_TASK_SOFT_TIME_LIMIT = 25 * 60 # 25 minutes
CELERY_TASK_ALWAYS_EAGER = False # Set to True for synchronous execution in development
# Celery Beat Configuration (for periodic tasks)
CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler"
# Celery Result Backend Configuration
CELERY_RESULT_EXTENDED = True
CELERY_RESULT_EXPIRES = 3600 # Results expire after 1 hour
# Celery Logging
CELERY_WORKER_HIJACK_ROOT_LOGGER = False
CELERY_WORKER_LOG_FORMAT = "[%(asctime)s: %(levelname)s/%(processName)s] %(message)s"
CELERY_WORKER_TASK_LOG_FORMAT = "[%(asctime)s: %(levelname)s/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s"
# Celery Accept Content
CELERY_ACCEPT_CONTENT = ["json"]
CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = "json"
CELERY_TIMEZONE = TIME_ZONE

View File

@@ -3,6 +3,8 @@ from .models import LyngSat
@admin.register(LyngSat)
class LyngSatAdmin(admin.ModelAdmin):
list_display = ("mark", "timestamp")
search_fields = ("mark", )
ordering = ("timestamp",)
list_display = ("id_satellite", "frequency", "polarization", "modulation", "last_update")
search_fields = ("id_satellite__name", "channel_info")
list_filter = ("id_satellite", "polarization", "modulation", "standard")
ordering = ("-last_update",)
readonly_fields = ("last_update",)

View File

@@ -0,0 +1,37 @@
# Generated by Django 5.2.7 on 2025-11-10 20:03
import django.db.models.deletion
import mainapp.models
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
('mainapp', '0007_remove_parameter_objitems_parameter_objitem'),
]
operations = [
migrations.CreateModel(
name='LyngSat',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('frequency', models.FloatField(blank=True, default=0, null=True, verbose_name='Частота, МГц')),
('sym_velocity', models.FloatField(blank=True, default=0, null=True, verbose_name='Символьная скорость, БОД')),
('last_update', models.DateTimeField(blank=True, null=True, verbose_name='Время')),
('channel_info', models.CharField(blank=True, max_length=20, null=True, verbose_name='Описание источника')),
('fec', models.CharField(blank=True, max_length=30, null=True, verbose_name='Коэффициент коррекции ошибок')),
('url', models.URLField(blank=True, null=True, verbose_name='Ссылка на страницу')),
('id_satellite', models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='lyngsat', to='mainapp.satellite', verbose_name='Спутник')),
('modulation', models.ForeignKey(blank=True, default=mainapp.models.get_default_modulation, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='lyngsat', to='mainapp.modulation', verbose_name='Модуляция')),
('polarization', models.ForeignKey(blank=True, default=mainapp.models.get_default_polarization, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='lyngsat', to='mainapp.polarization', verbose_name='Поляризация')),
('standard', models.ForeignKey(blank=True, default=mainapp.models.get_default_standard, null=True, on_delete=django.db.models.deletion.SET_DEFAULT, related_name='lyngsat', to='mainapp.standard', verbose_name='Стандарт')),
],
options={
'verbose_name': 'Источник LyngSat',
'verbose_name_plural': 'Источники LyngSat',
},
),
]

View File

@@ -4,76 +4,80 @@ from datetime import datetime
import re
import time
class LyngSatParser:
"""Парсер данных для LyngSat(Для работы нужен flaresolver)"""
def __init__(
self,
self,
flaresolver_url: str = "http://localhost:8191/v1",
regions: list[str] | None = None,
target_sats: list[str] | None = None,
):
self.flaresolver_url = flaresolver_url
self.regions = regions
self.target_sats = list(map(lambda sat: sat.strip().lower(), target_sats)) if regions else None
self.regions = regions if regions else ["europe", "asia", "america", "atlantic"]
self.target_sats = (
list(map(lambda sat: sat.strip().lower(), target_sats)) if regions else None
)
self.regions = regions if regions else ["europe", "asia", "america", "atlantic"]
self.BASE_URL = "https://www.lyngsat.com"
def parse_metadata(self, metadata: str) -> dict:
if not metadata or not metadata.strip():
return {
'standard': None,
'modulation': None,
'symbol_rate': None,
'fec': None
"standard": None,
"modulation": None,
"symbol_rate": None,
"fec": None,
}
normalized = re.sub(r'\s+', '', metadata.strip())
fec_match = re.search(r'([1-9]/[1-9])$', normalized)
normalized = re.sub(r"\s+", "", metadata.strip())
fec_match = re.search(r"([1-9]/[1-9])$", normalized)
fec = fec_match.group(1) if fec_match else None
if fec_match:
core = normalized[:fec_match.start()]
core = normalized[: fec_match.start()]
else:
core = normalized
std_match = re.match(r'(DVB-S2?|ABS-S|DVB-T2?|ATSC|ISDB)', core)
std_match = re.match(r"(DVB-S2?|ABS-S|DVB-T2?|ATSC|ISDB)", core)
standard = std_match.group(1) if std_match else None
rest = core[len(standard):] if standard else core
rest = core[len(standard) :] if standard else core
modulation = None
mod_match = re.match(r'(8PSK|QPSK|16APSK|32APSK|64QAM|256QAM|BPSK)', rest)
mod_match = re.match(r"(8PSK|QPSK|16APSK|32APSK|64QAM|256QAM|BPSK)", rest)
if mod_match:
modulation = mod_match.group(1)
rest = rest[len(modulation):]
rest = rest[len(modulation) :]
symbol_rate = None
sr_match = re.search(r'(\d+)$', rest)
sr_match = re.search(r"(\d+)$", rest)
if sr_match:
try:
symbol_rate = int(sr_match.group(1))
except ValueError:
pass
return {
'standard': standard,
'modulation': modulation,
'symbol_rate': symbol_rate,
'fec': fec
"standard": standard,
"modulation": modulation,
"symbol_rate": symbol_rate,
"fec": fec,
}
def extract_date(self, s: str) -> datetime | None:
s = s.strip()
match = re.search(r'(\d{6})$', s)
match = re.search(r"(\d{6})$", s)
if not match:
return None
yymmdd = match.group(1)
try:
return datetime.strptime(yymmdd, '%y%m%d').date()
return datetime.strptime(yymmdd, "%y%m%d").date()
except ValueError:
return None
def convert_polarization(self, polarization: str) -> str:
"""Преобразовать код поляризации в понятное название на русском"""
polarization_map = {
'V': 'Вертикальная',
'H': 'Горизонтальная',
'R': 'Правая',
'L': 'Левая'
"V": "Вертикальная",
"H": "Горизонтальная",
"R": "Правая",
"L": "Левая",
}
return polarization_map.get(polarization.upper(), polarization)
@@ -83,11 +87,7 @@ class LyngSatParser:
regions = self.regions
for region in regions:
url = f"{self.BASE_URL}/{region}.html"
payload = {
"cmd": "request.get",
"url": url,
"maxTimeout": 60000
}
payload = {"cmd": "request.get", "url": url, "maxTimeout": 60000}
response = requests.post(self.flaresolver_url, json=payload)
if response.status_code != 200:
continue
@@ -95,7 +95,7 @@ class LyngSatParser:
html_regions.append(html_content)
print(f"Обработал страницу по {region}")
return html_regions
def get_satellite_urls(self, html_regions: list[str]):
sat_names = []
sat_urls = []
@@ -104,19 +104,19 @@ class LyngSatParser:
col_table = soup.find_all("div", class_="desktab")[0]
tables = col_table.find_next_sibling('table').find_all('table')
tables = col_table.find_next_sibling("table").find_all("table")
trs = []
for table in tables:
trs.extend(table.find_all('tr'))
trs.extend(table.find_all("tr"))
for tr in trs:
sat_name = tr.find('span').text
sat_name = tr.find("span").text
if self.target_sats is not None:
if sat_name.strip().lower() not in self.target_sats:
continue
try:
sat_url = tr.find_all('a')[2]['href']
sat_url = tr.find_all("a")[2]["href"]
except IndexError:
sat_url = tr.find_all('a')[0]['href']
sat_url = tr.find_all("a")[0]["href"]
sat_names.append(sat_name)
sat_urls.append(sat_url)
return sat_names, sat_urls
@@ -128,60 +128,67 @@ class LyngSatParser:
col_table = soup.find_all("div", class_="desktab")[0]
tables = col_table.find_next_sibling('table').find_all('table')
tables = col_table.find_next_sibling("table").find_all("table")
trs = []
for table in tables:
trs.extend(table.find_all('tr'))
trs.extend(table.find_all("tr"))
for tr in trs:
sat_name = tr.find('span').text
sat_name = tr.find("span").text
if self.target_sats is not None:
if sat_name.strip().lower() not in self.target_sats:
continue
try:
sat_url = tr.find_all('a')[2]['href']
sat_url = tr.find_all("a")[2]["href"]
except IndexError:
sat_url = tr.find_all('a')[0]['href']
update_date = tr.find_all('td')[-1].text
sat_response = requests.post(self.flaresolver_url, json={
"cmd": "request.get",
"url": f"{self.BASE_URL}/{sat_url}",
"maxTimeout": 60000
})
html_content = sat_response.json().get("solution", {}).get("response", "")
sat_url = tr.find_all("a")[0]["href"]
update_date = tr.find_all("td")[-1].text
sat_response = requests.post(
self.flaresolver_url,
json={
"cmd": "request.get",
"url": f"{self.BASE_URL}/{sat_url}",
"maxTimeout": 60000,
},
)
html_content = (
sat_response.json().get("solution", {}).get("response", "")
)
sat_page_data = self.get_satellite_content(html_content)
sat_data[sat_name] = {
"url": f"{self.BASE_URL}/{sat_url}",
"update_date": datetime.strptime(update_date, "%y%m%d").date(),
"sources": sat_page_data
"sources": sat_page_data,
}
return sat_data
def get_satellite_content(self, html_content: str) -> dict:
sat_soup = BeautifulSoup(html_content, "html.parser")
big_table = sat_soup.find('table', class_='bigtable')
big_table = sat_soup.find("table", class_="bigtable")
all_tables = big_table.find_all("div", class_="desktab")[:-1]
data = []
for table in all_tables:
trs = table.find_next_sibling('table').find_all('tr')
trs = table.find_next_sibling("table").find_all("tr")
for idx, tr in enumerate(trs):
tds = tr.find_all('td')
tds = tr.find_all("td")
if len(tds) < 9 or idx < 2:
continue
freq, polarization = tds[0].find('b').text.strip().split('\xa0')
freq, polarization = tds[0].find("b").text.strip().split("\xa0")
polarization = self.convert_polarization(polarization)
meta = self.parse_metadata(tds[1].text)
provider_name = tds[3].text
last_update = self.extract_date(tds[-1].text)
data.append({
"freq": freq,
"pol": polarization,
"metadata": meta,
"provider_name": provider_name,
"last_update": last_update
})
data.append(
{
"freq": freq,
"pol": polarization,
"metadata": meta,
"provider_name": provider_name,
"last_update": last_update,
}
)
return data
class KingOfSatParser:
def __init__(self, base_url="https://ru.kingofsat.net", max_satellites=0):
@@ -193,20 +200,22 @@ class KingOfSatParser:
self.base_url = base_url
self.max_satellites = max_satellites
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
})
self.session.headers.update(
{
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
)
def convert_polarization(self, polarization):
"""Преобразовать код поляризации в понятное название на русском"""
polarization_map = {
'V': 'Вертикальная',
'H': 'Горизонтальная',
'R': 'Правая',
'L': 'Левая'
"V": "Вертикальная",
"H": "Горизонтальная",
"R": "Правая",
"L": "Левая",
}
return polarization_map.get(polarization.upper(), polarization)
def fetch_page(self, url):
"""Получить HTML страницу"""
try:
@@ -216,114 +225,112 @@ class KingOfSatParser:
except Exception as e:
print(f"Ошибка при получении страницы {url}: {e}")
return None
def parse_satellite_table(self, html_content):
"""Распарсить таблицу со спутниками"""
soup = BeautifulSoup(html_content, 'html.parser')
soup = BeautifulSoup(html_content, "html.parser")
satellites = []
table = soup.find('table')
table = soup.find("table")
if not table:
print("Таблица не найдена")
return satellites
rows = table.find_all('tr')[1:]
rows = table.find_all("tr")[1:]
for row in rows:
cols = row.find_all('td')
cols = row.find_all("td")
if len(cols) < 13:
continue
try:
position_cell = cols[0].text.strip()
position_match = re.search(r'([\d\.]+)°([EW])', position_cell)
position_match = re.search(r"([\d\.]+)°([EW])", position_cell)
if position_match:
position_value = position_match.group(1)
position_direction = position_match.group(2)
position = f"{position_value}{position_direction}"
else:
position = None
# Название спутника (2-я колонка)
satellite_cell = cols[1]
satellite_name = satellite_cell.get_text(strip=True)
# Удаляем возможные лишние символы или пробелы
satellite_name = re.sub(r'\s+', ' ', satellite_name).strip()
satellite_name = re.sub(r"\s+", " ", satellite_name).strip()
# NORAD (3-я колонка)
norad = cols[2].text.strip()
if not norad or norad == "-":
norad = None
ini_link = None
ini_cell = cols[3]
ini_img = ini_cell.find('img', src=lambda x: x and 'disquette.gif' in x)
ini_img = ini_cell.find("img", src=lambda x: x and "disquette.gif" in x)
if ini_img and position:
ini_link = f"https://ru.kingofsat.net/dl.php?pos={position}&fkhz=0"
update_date = cols[12].text.strip() if len(cols) > 12 else None
if satellite_name and ini_link and position:
satellites.append({
'position': position,
'name': satellite_name,
'norad': norad,
'ini_url': ini_link,
'update_date': update_date
})
satellites.append(
{
"position": position,
"name": satellite_name,
"norad": norad,
"ini_url": ini_link,
"update_date": update_date,
}
)
except Exception as e:
print(f"Ошибка при обработке строки таблицы: {e}")
continue
return satellites
def parse_ini_file(self, ini_content):
"""Распарсить содержимое .ini файла"""
data = {
'metadata': {},
'sattype': {},
'dvb': {}
}
data = {"metadata": {}, "sattype": {}, "dvb": {}}
# # Извлекаем метаданные из комментариев
# metadata_match = re.search(r'\[ downloaded from www\.kingofsat\.net \(c\) (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) \]', ini_content)
# if metadata_match:
# data['metadata']['downloaded'] = metadata_match.group(1)
# Парсим секцию [SATTYPE]
sattype_match = re.search(r'\[SATTYPE\](.*?)\n\[', ini_content, re.DOTALL)
sattype_match = re.search(r"\[SATTYPE\](.*?)\n\[", ini_content, re.DOTALL)
if sattype_match:
sattype_content = sattype_match.group(1).strip()
for line in sattype_content.split('\n'):
for line in sattype_content.split("\n"):
line = line.strip()
if '=' in line:
key, value = line.split('=', 1)
data['sattype'][key.strip()] = value.strip()
if "=" in line:
key, value = line.split("=", 1)
data["sattype"][key.strip()] = value.strip()
# Парсим секцию [DVB]
dvb_match = re.search(r'\[DVB\](.*?)(?:\n\[|$)', ini_content, re.DOTALL)
dvb_match = re.search(r"\[DVB\](.*?)(?:\n\[|$)", ini_content, re.DOTALL)
if dvb_match:
dvb_content = dvb_match.group(1).strip()
for line in dvb_content.split('\n'):
for line in dvb_content.split("\n"):
line = line.strip()
if '=' in line:
key, value = line.split('=', 1)
params = [p.strip() for p in value.split(',')]
polarization = params[1] if len(params) > 1 else ''
if "=" in line:
key, value = line.split("=", 1)
params = [p.strip() for p in value.split(",")]
polarization = params[1] if len(params) > 1 else ""
if polarization:
polarization = self.convert_polarization(polarization)
data['dvb'][key.strip()] = {
'frequency': params[0] if len(params) > 0 else '',
'polarization': polarization,
'symbol_rate': params[2] if len(params) > 2 else '',
'fec': params[3] if len(params) > 3 else '',
'standard': params[4] if len(params) > 4 else '',
'modulation': params[5] if len(params) > 5 else ''
data["dvb"][key.strip()] = {
"frequency": params[0] if len(params) > 0 else "",
"polarization": polarization,
"symbol_rate": params[2] if len(params) > 2 else "",
"fec": params[3] if len(params) > 3 else "",
"standard": params[4] if len(params) > 4 else "",
"modulation": params[5] if len(params) > 5 else "",
}
return data
def download_ini_file(self, url):
"""Скачать содержимое .ini файла"""
try:
@@ -333,71 +340,66 @@ class KingOfSatParser:
except Exception as e:
print(f"Ошибка при скачивании .ini файла {url}: {e}")
return None
def get_all_satellites_data(self):
"""Получить данные всех спутников с учетом ограничения max_satellites"""
html_content = self.fetch_page(self.base_url + '/satellites')
html_content = self.fetch_page(self.base_url + "/satellites")
if not html_content:
return []
satellites = self.parse_satellite_table(html_content)
if self.max_satellites > 0 and len(satellites) > self.max_satellites:
satellites = satellites[:self.max_satellites]
satellites = satellites[: self.max_satellites]
results = []
processed_count = 0
for satellite in satellites:
print(f"Обработка спутника: {satellite['name']} ({satellite['position']})")
ini_content = self.download_ini_file(satellite['ini_url'])
ini_content = self.download_ini_file(satellite["ini_url"])
if not ini_content:
print(f"Не удалось скачать .ini файл для {satellite['name']}")
continue
parsed_ini = self.parse_ini_file(ini_content)
result = {
'satellite_name': satellite['name'],
'position': satellite['position'],
'norad': satellite['norad'],
'update_date': satellite['update_date'],
'ini_url': satellite['ini_url'],
'ini_data': parsed_ini
"satellite_name": satellite["name"],
"position": satellite["position"],
"norad": satellite["norad"],
"update_date": satellite["update_date"],
"ini_url": satellite["ini_url"],
"ini_data": parsed_ini,
}
results.append(result)
processed_count += 1
if self.max_satellites > 0 and processed_count >= self.max_satellites:
break
time.sleep(1)
time.sleep(1)
return results
def create_satellite_dict(self, satellites_data):
"""Создать словарь с данными спутников"""
satellite_dict = {}
for data in satellites_data:
key = f"{data['position']}_{data['satellite_name'].replace(' ', '_').replace('/', '_')}"
satellite_dict[key] = {
'name': data['satellite_name'],
'position': data['position'],
'norad': data['norad'],
'update_date': data['update_date'],
'ini_url': data['ini_url'],
'transponders_count': len(data['ini_data']['dvb']),
'transponders': data['ini_data']['dvb'],
'sattype_info': data['ini_data']['sattype'],
'metadata': data['ini_data']['metadata']
"name": data["satellite_name"],
"position": data["position"],
"norad": data["norad"],
"update_date": data["update_date"],
"ini_url": data["ini_url"],
"transponders_count": len(data["ini_data"]["dvb"]),
"transponders": data["ini_data"]["dvb"],
"sattype_info": data["ini_data"]["sattype"],
"metadata": data["ini_data"]["metadata"],
}
return satellite_dict
from pprint import pprint
lyngsat = LyngSatParser(regions=['europe'], target_sats=['Türksat 3A', 'Intelsat 22'])
html_regions = lyngsat.get_region_pages()
pprint(lyngsat.get_satellite_urls(html_regions))
return satellite_dict

73
dbapp/lyngsatapp/tasks.py Normal file
View File

@@ -0,0 +1,73 @@
"""
Celery tasks for Lyngsat data processing.
"""
import logging
from celery import shared_task
from django.core.cache import cache
from .utils import fill_lyngsat_data
logger = logging.getLogger(__name__)
@shared_task(bind=True, name='lyngsatapp.fill_lyngsat_data_async')
def fill_lyngsat_data_task(self, target_sats, regions=None):
"""
Асинхронная задача для заполнения данных Lyngsat.
Args:
target_sats: Список названий спутников для обработки
regions: Список регионов для парсинга (по умолчанию все)
Returns:
dict: Статистика обработки
"""
task_id = self.request.id
logger.info(f"[Task {task_id}] Начало обработки данных Lyngsat")
logger.info(f"[Task {task_id}] Спутники: {', '.join(target_sats)}")
logger.info(f"[Task {task_id}] Регионы: {', '.join(regions) if regions else 'все'}")
# Обновляем статус задачи
self.update_state(
state='PROGRESS',
meta={
'current': 0,
'total': len(target_sats),
'status': 'Инициализация...'
}
)
try:
# Вызываем функцию заполнения данных
stats = fill_lyngsat_data(
target_sats=target_sats,
regions=regions,
task_id=task_id,
update_progress=lambda current, total, status: self.update_state(
state='PROGRESS',
meta={
'current': current,
'total': total,
'status': status
}
)
)
logger.info(f"[Task {task_id}] Обработка завершена успешно")
logger.info(f"[Task {task_id}] Статистика: {stats}")
# Сохраняем результат в кеш для отображения на странице
cache.set(f'lyngsat_task_{task_id}', stats, timeout=3600)
return stats
except Exception as e:
logger.error(f"[Task {task_id}] Ошибка при обработке: {str(e)}", exc_info=True)
self.update_state(
state='FAILURE',
meta={
'error': str(e),
'status': 'Ошибка при обработке'
}
)
raise

View File

@@ -1,58 +1,170 @@
import logging
from .parser import LyngSatParser
from .models import LyngSat
from mainapp.models import Polarization, Standard, Modulation, Satellite
def fill_lyngsat_data(target_sats: list[str]):
parser = LyngSatParser(
target_sats=target_sats,
)
lyngsat_data = parser.get_satellites_data()
for sat_name, data in lyngsat_data.items():
url = data['url']
sources = data['sources']
for source in sources:
logger = logging.getLogger(__name__)
def fill_lyngsat_data(
target_sats: list[str],
regions: list[str] = None,
task_id: str = None,
update_progress=None
):
"""
Заполняет данные Lyngsat для указанных спутников и регионов.
Args:
target_sats: Список названий спутников для обработки
regions: Список регионов для парсинга (по умолчанию все)
task_id: ID задачи Celery для логирования
update_progress: Функция для обновления прогресса (current, total, status)
Returns:
dict: Статистика обработки с ключами:
- total_satellites: общее количество спутников
- total_sources: общее количество источников
- created: количество созданных записей
- updated: количество обновленных записей
- errors: список ошибок
"""
log_prefix = f"[Task {task_id}]" if task_id else "[Lyngsat]"
stats = {
'total_satellites': 0,
'total_sources': 0,
'created': 0,
'updated': 0,
'errors': []
}
if regions is None:
regions = ["europe", "asia", "america", "atlantic"]
logger.info(f"{log_prefix} Начало парсинга данных")
logger.info(f"{log_prefix} Спутники: {', '.join(target_sats)}")
logger.info(f"{log_prefix} Регионы: {', '.join(regions)}")
if update_progress:
update_progress(0, len(target_sats), "Инициализация парсера...")
try:
parser = LyngSatParser(
target_sats=target_sats,
regions=regions
)
logger.info(f"{log_prefix} Получение данных со спутников...")
if update_progress:
update_progress(0, len(target_sats), "Получение данных со спутников...")
lyngsat_data = parser.get_satellites_data()
stats['total_satellites'] = len(lyngsat_data)
logger.info(f"{log_prefix} Получено данных по {stats['total_satellites']} спутникам")
for idx, (sat_name, data) in enumerate(lyngsat_data.items(), 1):
logger.info(f"{log_prefix} Обработка спутника {idx}/{stats['total_satellites']}: {sat_name}")
if update_progress:
update_progress(idx, stats['total_satellites'], f"Обработка {sat_name}...")
url = data['url']
sources = data['sources']
stats['total_sources'] += len(sources)
logger.info(f"{log_prefix} Найдено {len(sources)} источников для {sat_name}")
# Находим спутник в базе
try:
freq = float(source['freq'])
except Exception as e:
freq = -1.0
print("Беда с частотой")
last_update = source['last_update']
fec = source['metadata']['fec']
modulation = source['metadata']['modulation']
standard = source['metadata']['standard']
symbol_velocity = source['metadata']['symbol_rate']
polarization = source['pol']
channel_info = source['provider_name']
sat_obj = Satellite.objects.get(name__icontains=sat_name)
logger.debug(f"{log_prefix} Спутник {sat_name} найден в базе (ID: {sat_obj.id})")
except Satellite.DoesNotExist:
error_msg = f"Спутник '{sat_name}' не найден в базе данных"
logger.warning(f"{log_prefix} {error_msg}")
stats['errors'].append(error_msg)
continue
except Satellite.MultipleObjectsReturned:
error_msg = f"Найдено несколько спутников с именем '{sat_name}'"
logger.warning(f"{log_prefix} {error_msg}")
stats['errors'].append(error_msg)
continue
for source_idx, source in enumerate(sources, 1):
try:
# Парсим частоту
try:
freq = float(source['freq'])
except (ValueError, TypeError):
freq = -1.0
error_msg = f"Некорректная частота для {sat_name}: {source.get('freq')}"
logger.debug(f"{log_prefix} {error_msg}")
stats['errors'].append(error_msg)
last_update = source['last_update']
fec = source['metadata'].get('fec')
modulation_name = source['metadata'].get('modulation')
standard_name = source['metadata'].get('standard')
symbol_velocity = source['metadata'].get('symbol_rate')
polarization_name = source['pol']
channel_info = source['provider_name']
pol_obj, _ = Polarization.objects.get_or_create(
name=polarization
)
# Создаем или получаем связанные объекты
pol_obj, _ = Polarization.objects.get_or_create(
name=polarization_name if polarization_name else "-"
)
mod_obj, _ = Modulation.objects.get_or_create(
name=modulation
)
mod_obj, _ = Modulation.objects.get_or_create(
name=modulation_name if modulation_name else "-"
)
standard_obj, _ = Standard.objects.get_or_create(
name=standard
)
standard_obj, _ = Standard.objects.get_or_create(
name=standard_name if standard_name else "-"
)
sat_obj, _ = Satellite.objects.get(
name__contains=sat_name
)
lyng_obj, _ = LyngSat.objects.get_or_create(
id_satellite=sat_obj,
frequency=freq,
polarization=pol_obj,
defaults={
"modulation": mod_obj,
"standard": standard_obj,
"sym_velocity": symbol_velocity,
"channel_info": channel_info,
"last_update": last_update,
"fec": fec,
"url": url
}
)
lyng_obj.objects.update_or_create()
# TODO: сделать карточку и форму для действий и выбора спутника
lyng_obj.save()
# Создаем или обновляем запись Lyngsat
lyng_obj, created = LyngSat.objects.update_or_create(
id_satellite=sat_obj,
frequency=freq,
polarization=pol_obj,
defaults={
"modulation": mod_obj,
"standard": standard_obj,
"sym_velocity": symbol_velocity if symbol_velocity else 0,
"channel_info": channel_info[:20] if channel_info else "",
"last_update": last_update,
"fec": fec[:30] if fec else "",
"url": url
}
)
if created:
stats['created'] += 1
logger.debug(f"{log_prefix} Создана запись для {sat_name} {freq} МГц")
else:
stats['updated'] += 1
logger.debug(f"{log_prefix} Обновлена запись для {sat_name} {freq} МГц")
# Логируем прогресс каждые 10 источников
if source_idx % 10 == 0:
logger.info(f"{log_prefix} Обработано {source_idx}/{len(sources)} источников для {sat_name}")
except Exception as e:
error_msg = f"Ошибка при обработке источника {sat_name}: {str(e)}"
logger.error(f"{log_prefix} {error_msg}", exc_info=True)
stats['errors'].append(error_msg)
continue
logger.info(f"{log_prefix} Завершена обработка {sat_name}: создано {stats['created']}, обновлено {stats['updated']}")
except Exception as e:
error_msg = f"Критическая ошибка: {str(e)}"
logger.error(f"{log_prefix} {error_msg}", exc_info=True)
stats['errors'].append(error_msg)
logger.info(f"{log_prefix} Обработка завершена. Итого: создано {stats['created']}, обновлено {stats['updated']}, ошибок {len(stats['errors'])}")
if update_progress:
update_progress(stats['total_satellites'], stats['total_satellites'], "Завершено")
return stats

View File

@@ -107,6 +107,40 @@ class NewEventForm(forms.Form):
'accept': '.xlsx,.xls'
})
)
class FillLyngsatDataForm(forms.Form):
"""Форма для заполнения данных из Lyngsat"""
REGION_CHOICES = [
('europe', 'Европа'),
('asia', 'Азия'),
('america', 'Америка'),
('atlantic', 'Атлантика'),
]
satellites = forms.ModelMultipleChoiceField(
queryset=Satellite.objects.all().order_by('name'),
label="Выберите спутники",
widget=forms.SelectMultiple(attrs={
'class': 'form-select',
'size': '10'
}),
required=True,
help_text="Удерживайте Ctrl (Cmd на Mac) для выбора нескольких спутников"
)
regions = forms.MultipleChoiceField(
choices=REGION_CHOICES,
label="Выберите регионы",
widget=forms.SelectMultiple(attrs={
'class': 'form-select',
'size': '4'
}),
required=True,
initial=['europe', 'asia', 'america', 'atlantic'],
help_text="Удерживайте Ctrl (Cmd на Mac) для выбора нескольких регионов"
)
class ParameterForm(forms.ModelForm):
"""
Форма для создания и редактирования параметров ВЧ загрузки.

View File

@@ -124,23 +124,23 @@
</div>
</div>
<!-- Map Views Card -->
<!-- Lyngsat Data Fill Card -->
<div class="col-lg-6">
<div class="card h-100 shadow-sm border-0">
<div class="card-body">
<div class="d-flex align-items-center mb-3">
<div class="bg-secondary bg-opacity-10 rounded-circle p-2 me-3">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-map text-secondary" viewBox="0 0 16 16">
<path d="M15.817.113A.5.5 0 0 1 16 .5v14a.5.5 0 0 1-.402.49l-5 1a.502.502 0 0 1-.196 0L5.5 15.01l-4.902.98A.5.5 0 0 1 0 15.5v-14a.5.5 0 0 1 .402-.49l5-1a.5.5 0 0 1 .196 0L10.5.99l4.902-.98a.5.5 0 0 1 .415.103M10 1.91l-4-.8v12.98l4 .8zM1.61 2.22l4.39.88v10.88l-4.39-.88zm9.18 10.88 4-.8V2.34l-4 .8z"/>
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-cloud-download text-secondary" viewBox="0 0 16 16">
<path d="M4.406 1.342A5.53 5.53 0 0 1 8 0c2.69 0 4.923 2 5.166 4.579C14.758 4.804 16 6.137 16 7.773 16 9.569 14.502 11 12.687 11H10a.5.5 0 0 1 0-1h2.688C13.979 10 15 8.988 15 7.773c0-1.216-1.02-2.228-2.313-2.228h-.5v-.5C12.188 2.825 10.328 1 8 1a4.53 4.53 0 0 0-2.941 1.1c-.757.652-1.153 1.438-1.153 2.055v.448l-.445.049C2.064 4.805 1 5.952 1 7.318 1 8.785 2.23 10 3.781 10H6a.5.5 0 0 1 0 1H3.781C1.708 11 0 9.366 0 7.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383"/>
<path d="M7.646 15.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 14.293V5.5a.5.5 0 0 0-1 0v8.793l-2.146-2.147a.5.5 0 0 0-.708.708z"/>
</svg>
</div>
<h3 class="card-title mb-0">Карты</h3>
</div>
<p class="card-text">Просматривайте данные на 2D и 3D картах для визуализации геолокации спутников.</p>
<div class="mt-2">
<a href="{% url 'mapsapp:2dmap' %}" class="btn btn-secondary me-2">2D Карта</a>
<a href="{% url 'mapsapp:3dmap' %}" class="btn btn-outline-secondary">3D Карта</a>
<h3 class="card-title mb-0">Заполнение данных Lyngsat</h3>
</div>
<p class="card-text">Загрузите данные о транспондерах спутников с сайта Lyngsat. Выберите спутники и регионы для парсинга данных.</p>
<a href="{% url 'mainapp:fill_lyngsat_data' %}" class="btn btn-secondary">
Заполнить данные Lyngsat
</a>
</div>
</div>
</div>

View File

@@ -0,0 +1,118 @@
{% extends 'mainapp/base.html' %}
{% block title %}Заполнение данных Lyngsat{% endblock %}
{% block content %}
<div class="container mt-4">
<div class="row justify-content-center">
<div class="col-lg-8">
<div class="card shadow-sm">
<div class="card-header bg-primary text-white">
<h3 class="mb-0">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-cloud-download me-2" viewBox="0 0 16 16">
<path d="M4.406 1.342A5.53 5.53 0 0 1 8 0c2.69 0 4.923 2 5.166 4.579C14.758 4.804 16 6.137 16 7.773 16 9.569 14.502 11 12.687 11H10a.5.5 0 0 1 0-1h2.688C13.979 10 15 8.988 15 7.773c0-1.216-1.02-2.228-2.313-2.228h-.5v-.5C12.188 2.825 10.328 1 8 1a4.53 4.53 0 0 0-2.941 1.1c-.757.652-1.153 1.438-1.153 2.055v.448l-.445.049C2.064 4.805 1 5.952 1 7.318 1 8.785 2.23 10 3.781 10H6a.5.5 0 0 1 0 1H3.781C1.708 11 0 9.366 0 7.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383"/>
<path d="M7.646 15.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 14.293V5.5a.5.5 0 0 0-1 0v8.793l-2.146-2.147a.5.5 0 0 0-.708.708z"/>
</svg>
Заполнение данных из Lyngsat
</h3>
</div>
<div class="card-body">
<!-- Alert messages -->
{% include 'mainapp/components/_messages.html' %}
<div class="alert alert-info" role="alert">
<strong>Внимание!</strong> Процесс заполнения данных может занять продолжительное время,
так как выполняются запросы к внешнему сайту Lyngsat. Пожалуйста, дождитесь завершения операции.
</div>
<form method="post" class="needs-validation" novalidate>
{% csrf_token %}
<!-- Satellites Selection -->
<div class="mb-4">
<label for="{{ form.satellites.id_for_label }}" class="form-label fw-bold">
{{ form.satellites.label }}
</label>
{{ form.satellites }}
{% if form.satellites.help_text %}
<div class="form-text">{{ form.satellites.help_text }}</div>
{% endif %}
{% if form.satellites.errors %}
<div class="invalid-feedback d-block">
{{ form.satellites.errors }}
</div>
{% endif %}
</div>
<!-- Regions Selection -->
<div class="mb-4">
<label for="{{ form.regions.id_for_label }}" class="form-label fw-bold">
{{ form.regions.label }}
</label>
{{ form.regions }}
{% if form.regions.help_text %}
<div class="form-text">{{ form.regions.help_text }}</div>
{% endif %}
{% if form.regions.errors %}
<div class="invalid-feedback d-block">
{{ form.regions.errors }}
</div>
{% endif %}
</div>
<!-- Buttons -->
<div class="d-grid gap-2 d-md-flex justify-content-md-between">
<a href="{% url 'mainapp:actions' %}" class="btn btn-secondary">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-arrow-left me-1" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M15 8a.5.5 0 0 0-.5-.5H2.707l3.147-3.146a.5.5 0 1 0-.708-.708l-4 4a.5.5 0 0 0 0 .708l4 4a.5.5 0 0 0 .708-.708L2.707 8.5H14.5A.5.5 0 0 0 15 8"/>
</svg>
Назад
</a>
<button type="submit" class="btn btn-primary">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-download me-1" viewBox="0 0 16 16">
<path d="M.5 9.9a.5.5 0 0 1 .5.5v2.5a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1v-2.5a.5.5 0 0 1 1 0v2.5a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2v-2.5a.5.5 0 0 1 .5-.5"/>
<path d="M7.646 11.854a.5.5 0 0 0 .708 0l3-3a.5.5 0 0 0-.708-.708L8.5 10.293V1.5a.5.5 0 0 0-1 0v8.793L5.354 8.146a.5.5 0 1 0-.708.708z"/>
</svg>
Заполнить данные
</button>
</div>
</form>
</div>
</div>
<!-- Info Card -->
<div class="card mt-4 shadow-sm">
<div class="card-body">
<h5 class="card-title">Информация</h5>
<p class="card-text">
Эта форма позволяет загрузить данные о транспондерах спутников с сайта Lyngsat.
Выберите один или несколько спутников и регионы для парсинга данных.
</p>
<ul>
<li>Данные включают частоты, поляризацию, модуляцию, стандарты и другие параметры</li>
<li>Процесс может занять несколько минут в зависимости от количества выбранных спутников</li>
<li>Существующие записи будут обновлены, новые - созданы</li>
</ul>
</div>
</div>
</div>
</div>
</div>
<script>
// Form validation
(function() {
'use strict';
var forms = document.querySelectorAll('.needs-validation');
Array.prototype.slice.call(forms).forEach(function(form) {
form.addEventListener('submit', function(event) {
if (!form.checkValidity()) {
event.preventDefault();
event.stopPropagation();
}
form.classList.add('was-validated');
}, false);
});
})();
</script>
{% endblock %}

View File

@@ -0,0 +1,241 @@
{% extends 'mainapp/base.html' %}
{% block title %}Статус задачи Lyngsat{% endblock %}
{% block content %}
<div class="container mt-4">
<div class="row justify-content-center">
<div class="col-lg-8">
<div class="card shadow-sm">
<div class="card-header bg-primary text-white">
<h3 class="mb-0">
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="bi bi-hourglass-split me-2" viewBox="0 0 16 16">
<path d="M2.5 15a.5.5 0 1 1 0-1h1v-1a4.5 4.5 0 0 1 2.557-4.06c.29-.139.443-.377.443-.59v-.7c0-.213-.154-.451-.443-.59A4.5 4.5 0 0 1 3.5 3V2h-1a.5.5 0 0 1 0-1h11a.5.5 0 0 1 0 1h-1v1a4.5 4.5 0 0 1-2.557 4.06c-.29.139-.443.377-.443.59v.7c0 .213.154.451.443.59A4.5 4.5 0 0 1 12.5 13v1h1a.5.5 0 0 1 0 1zm2-13v1c0 .537.12 1.045.337 1.5h6.326c.216-.455.337-.963.337-1.5V2zm3 6.35c0 .701-.478 1.236-1.011 1.492A3.5 3.5 0 0 0 4.5 13s.866-1.299 3-1.48zm1 0v3.17c2.134.181 3 1.48 3 1.48a3.5 3.5 0 0 0-1.989-3.158C8.978 9.586 8.5 9.052 8.5 8.351z"/>
</svg>
Статус задачи заполнения данных Lyngsat
</h3>
</div>
<div class="card-body">
{% if task_id %}
<div class="mb-3">
<strong>ID задачи:</strong> <code id="task-id">{{ task_id }}</code>
</div>
<!-- Progress Bar -->
<div class="mb-4">
<div class="d-flex justify-content-between mb-2">
<span id="status-text">Загрузка статуса...</span>
<span id="progress-percent">0%</span>
</div>
<div class="progress" style="height: 25px;">
<div id="progress-bar" class="progress-bar progress-bar-striped progress-bar-animated"
role="progressbar" style="width: 0%;"
aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
<span id="progress-text">0%</span>
</div>
</div>
</div>
<!-- Task State -->
<div id="task-state-container" class="alert alert-info" role="alert">
<strong>Состояние:</strong> <span id="task-state">Проверка...</span>
</div>
<!-- Results Container (hidden by default) -->
<div id="results-container" class="d-none">
<h5 class="mt-4">Результаты обработки</h5>
<div class="row">
<div class="col-md-6">
<div class="card mb-3">
<div class="card-body">
<h6 class="card-subtitle mb-2 text-muted">Обработано спутников</h6>
<h3 class="card-title" id="result-satellites">-</h3>
</div>
</div>
</div>
<div class="col-md-6">
<div class="card mb-3">
<div class="card-body">
<h6 class="card-subtitle mb-2 text-muted">Обработано источников</h6>
<h3 class="card-title" id="result-sources">-</h3>
</div>
</div>
</div>
<div class="col-md-6">
<div class="card mb-3 border-success">
<div class="card-body">
<h6 class="card-subtitle mb-2 text-success">Создано записей</h6>
<h3 class="card-title text-success" id="result-created">-</h3>
</div>
</div>
</div>
<div class="col-md-6">
<div class="card mb-3 border-info">
<div class="card-body">
<h6 class="card-subtitle mb-2 text-info">Обновлено записей</h6>
<h3 class="card-title text-info" id="result-updated">-</h3>
</div>
</div>
</div>
</div>
<!-- Errors -->
<div id="errors-container" class="d-none">
<h6 class="text-danger">Ошибки при обработке:</h6>
<div class="alert alert-warning">
<ul id="errors-list" class="mb-0"></ul>
</div>
</div>
</div>
<!-- Error Container (hidden by default) -->
<div id="error-container" class="alert alert-danger d-none" role="alert">
<strong>Ошибка:</strong> <span id="error-text"></span>
</div>
<!-- Action Buttons -->
<div class="d-grid gap-2 d-md-flex justify-content-md-between mt-4">
<a href="{% url 'mainapp:fill_lyngsat_data' %}" class="btn btn-secondary">
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-arrow-left me-1" viewBox="0 0 16 16">
<path fill-rule="evenodd" d="M15 8a.5.5 0 0 0-.5-.5H2.707l3.147-3.146a.5.5 0 1 0-.708-.708l-4 4a.5.5 0 0 0 0 .708l4 4a.5.5 0 0 0 .708-.708L2.707 8.5H14.5A.5.5 0 0 0 15 8"/>
</svg>
Назад к форме
</a>
<a href="{% url 'mainapp:actions' %}" class="btn btn-outline-primary" id="actions-btn">
Перейти к действиям
</a>
</div>
{% else %}
<div class="alert alert-warning" role="alert">
ID задачи не указан. Пожалуйста, запустите задачу через форму заполнения данных.
</div>
<a href="{% url 'mainapp:fill_lyngsat_data' %}" class="btn btn-primary">
Перейти к форме
</a>
{% endif %}
</div>
</div>
</div>
</div>
</div>
{% if task_id %}
<script>
let taskId = '{{ task_id }}';
let pollInterval;
let isCompleted = false;
function updateProgress(data) {
const statusText = document.getElementById('status-text');
const progressBar = document.getElementById('progress-bar');
const progressText = document.getElementById('progress-text');
const progressPercent = document.getElementById('progress-percent');
const taskState = document.getElementById('task-state');
const taskStateContainer = document.getElementById('task-state-container');
// Update state
taskState.textContent = data.state;
if (data.state === 'PENDING') {
statusText.textContent = 'Задача в очереди...';
taskStateContainer.className = 'alert alert-info';
} else if (data.state === 'PROGRESS') {
const percent = data.percent || 0;
statusText.textContent = data.status || 'Обработка...';
progressBar.style.width = percent + '%';
progressBar.setAttribute('aria-valuenow', percent);
progressText.textContent = percent + '%';
progressPercent.textContent = percent + '%';
taskStateContainer.className = 'alert alert-info';
} else if (data.state === 'SUCCESS') {
statusText.textContent = 'Задача завершена успешно!';
progressBar.style.width = '100%';
progressBar.setAttribute('aria-valuenow', 100);
progressText.textContent = '100%';
progressPercent.textContent = '100%';
progressBar.classList.remove('progress-bar-animated');
progressBar.classList.add('bg-success');
taskStateContainer.className = 'alert alert-success';
// Show results
if (data.result) {
showResults(data.result);
}
isCompleted = true;
clearInterval(pollInterval);
} else if (data.state === 'FAILURE') {
statusText.textContent = 'Ошибка при выполнении задачи';
progressBar.classList.remove('progress-bar-animated');
progressBar.classList.add('bg-danger');
taskStateContainer.className = 'alert alert-danger';
// Show error
const errorContainer = document.getElementById('error-container');
const errorText = document.getElementById('error-text');
errorText.textContent = data.error || 'Неизвестная ошибка';
errorContainer.classList.remove('d-none');
isCompleted = true;
clearInterval(pollInterval);
}
}
function showResults(result) {
const resultsContainer = document.getElementById('results-container');
resultsContainer.classList.remove('d-none');
document.getElementById('result-satellites').textContent = result.total_satellites || 0;
document.getElementById('result-sources').textContent = result.total_sources || 0;
document.getElementById('result-created').textContent = result.created || 0;
document.getElementById('result-updated').textContent = result.updated || 0;
// Show errors if any
if (result.errors && result.errors.length > 0) {
const errorsContainer = document.getElementById('errors-container');
const errorsList = document.getElementById('errors-list');
errorsContainer.classList.remove('d-none');
errorsList.innerHTML = '';
result.errors.slice(0, 10).forEach(error => {
const li = document.createElement('li');
li.textContent = error;
errorsList.appendChild(li);
});
if (result.errors.length > 10) {
const li = document.createElement('li');
li.textContent = `И еще ${result.errors.length - 10} ошибок...`;
li.className = 'text-muted';
errorsList.appendChild(li);
}
}
}
function checkTaskStatus() {
fetch(`/api/lyngsat-task-status/${taskId}/`)
.then(response => response.json())
.then(data => {
updateProgress(data);
})
.catch(error => {
console.error('Error checking task status:', error);
});
}
// Start polling
document.addEventListener('DOMContentLoaded', function() {
checkTaskStatus();
pollInterval = setInterval(checkTaskStatus, 2000); // Poll every 2 seconds
// Stop polling after 30 minutes
setTimeout(() => {
if (!isCompleted) {
clearInterval(pollInterval);
document.getElementById('status-text').textContent = 'Превышено время ожидания. Обновите страницу для проверки статуса.';
}
}, 30 * 60 * 1000);
});
</script>
{% endif %}
{% endblock %}

View File

@@ -25,4 +25,8 @@ urlpatterns = [
path('object/<int:pk>/edit/', views.ObjItemUpdateView.as_view(), name='objitem_update'),
path('object/<int:pk>/', views.ObjItemDetailView.as_view(), name='objitem_detail'),
path('object/<int:pk>/delete/', views.ObjItemDeleteView.as_view(), name='objitem_delete'),
path('fill-lyngsat-data/', views.FillLyngsatDataView.as_view(), name='fill_lyngsat_data'),
path('lyngsat-task-status/', views.LyngsatTaskStatusView.as_view(), name='lyngsat_task_status'),
path('lyngsat-task-status/<str:task_id>/', views.LyngsatTaskStatusView.as_view(), name='lyngsat_task_status'),
path('api/lyngsat-task-status/<str:task_id>/', views.LyngsatTaskStatusAPIView.as_view(), name='lyngsat_task_status_api'),
]

View File

@@ -37,6 +37,7 @@ from .forms import (
UploadFileForm,
UploadVchLoad,
VchLinkForm,
FillLyngsatDataForm,
)
from .mixins import CoordinateProcessingMixin, FormMessageMixin, RoleRequiredMixin
from .models import Geo, Modulation, ObjItem, Polarization, Satellite
@@ -1029,3 +1030,97 @@ class ObjItemDetailView(LoginRequiredMixin, View):
}
return render(request, "mainapp/objitem_detail.html", context)
class FillLyngsatDataView(LoginRequiredMixin, FormMessageMixin, FormView):
"""
Представление для заполнения данных из Lyngsat.
Позволяет выбрать спутники и регионы для парсинга данных с сайта Lyngsat.
Запускает асинхронную задачу Celery для обработки.
"""
template_name = "mainapp/fill_lyngsat_data.html"
form_class = FillLyngsatDataForm
success_url = reverse_lazy("mainapp:lyngsat_task_status")
error_message = "Форма заполнена некорректно"
def form_valid(self, form):
satellites = form.cleaned_data["satellites"]
regions = form.cleaned_data["regions"]
# Получаем названия спутников
target_sats = [sat.name for sat in satellites]
try:
from lyngsatapp.tasks import fill_lyngsat_data_task
# Запускаем асинхронную задачу
task = fill_lyngsat_data_task.delay(target_sats, regions)
messages.success(
self.request,
f"Задача запущена! ID задачи: {task.id}. "
"Вы будете перенаправлены на страницу отслеживания прогресса."
)
# Перенаправляем на страницу статуса задачи
return redirect('mainapp:lyngsat_task_status', task_id=task.id)
except Exception as e:
messages.error(self.request, f"Ошибка при запуске задачи: {str(e)}")
return redirect("mainapp:fill_lyngsat_data")
class LyngsatTaskStatusView(LoginRequiredMixin, View):
"""
Представление для отслеживания статуса задачи заполнения данных Lyngsat.
"""
template_name = "mainapp/lyngsat_task_status.html"
def get(self, request, task_id=None):
context = {
'task_id': task_id
}
return render(request, self.template_name, context)
class LyngsatTaskStatusAPIView(LoginRequiredMixin, View):
"""
API для получения статуса задачи Celery.
"""
def get(self, request, task_id):
from celery.result import AsyncResult
from django.core.cache import cache
task = AsyncResult(task_id)
response_data = {
'task_id': task_id,
'state': task.state,
'result': None,
'error': None
}
if task.state == 'PENDING':
response_data['status'] = 'Задача в очереди...'
elif task.state == 'PROGRESS':
response_data['status'] = task.info.get('status', '')
response_data['current'] = task.info.get('current', 0)
response_data['total'] = task.info.get('total', 1)
response_data['percent'] = int((task.info.get('current', 0) / task.info.get('total', 1)) * 100)
elif task.state == 'SUCCESS':
# Получаем результат из кеша
result = cache.get(f'lyngsat_task_{task_id}')
if result:
response_data['result'] = result
response_data['status'] = 'Задача завершена успешно'
else:
response_data['result'] = task.result
response_data['status'] = 'Задача завершена'
elif task.state == 'FAILURE':
response_data['status'] = 'Ошибка при выполнении задачи'
response_data['error'] = str(task.info)
else:
response_data['status'] = task.state
return JsonResponse(response_data)

View File

@@ -24,6 +24,8 @@ pandas>=2.3.3
psycopg>=3.2.10
psycopg2-binary>=2.9.11
redis>=6.4.0
celery>=5.4.0
django-celery-results>=2.5.1
requests>=2.32.5
reverse-geocoder>=1.5.1
scikit-learn>=1.7.2

5
dbapp/start_celery_worker.sh Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/bash
# Script to start Celery worker
echo "Starting Celery worker..."
celery -A dbapp worker --loglevel=info --logfile=logs/celery_worker.log