Datadog入門 第10部 - トラブルシューティング・応用編と継続的成長の完全ガイド ​

# 問題診断チェックリスト自動実行スクリプト
#!/usr/bin/env python3

import subprocess
import json
import requests
from datetime import datetime, timedelta

class DatadogTroubleshooter:
    def __init__(self, api_key, app_key):
        self.api_key = api_key
        self.app_key = app_key
        self.base_url = "https://api.datadoghq.com/api/v1"
        
    def step1_connectivity_check(self):
        """ステップ1: 基本接続性確認"""
        print("📡 ステップ1: 接続性診断")
        
        # Agent接続状況確認
        try:
            response = requests.get(
                f"{self.base_url}/validate",
                params={"api_key": self.api_key, "application_key": self.app_key}
            )
            if response.status_code == 200:
                print("✅ API接続: 正常")
            else:
                print(f"❌ API接続エラー: {response.status_code}")
                return False
        except Exception as e:
            print(f"❌ 接続例外: {e}")
            return False
        
        # Agent status確認
        agent_status = subprocess.run(['sudo', 'datadog-agent', 'status'], 
                                    capture_output=True, text=True)
        if agent_status.returncode == 0:
            print("✅ Agent: 稼働中")
            return True
        else:
            print("❌ Agent: 異常")
            return False
    
    def step2_data_flow_analysis(self):
        """ステップ2: データフロー分析"""
        print("📊 ステップ2: データフロー診断")
        
        # メトリクス取得状況確認
        current_time = int(datetime.now().timestamp())
        start_time = current_time - 3600  # 1時間前
        
        try:
            response = requests.get(
                f"{self.base_url}/query",
                params={
                    "api_key": self.api_key,
                    "application_key": self.app_key,
                    "query": "avg:system.cpu.idle{*}",
                    "from": start_time,
                    "to": current_time
                }
            )
            
            if response.status_code == 200:
                data = response.json()
                if data.get('series') and len(data['series']) > 0:
                    print("✅ メトリクス収集: 正常")
                    print(f"   データポイント数: {len(data['series'][0].get('pointlist', []))}")
                else:
                    print("⚠️ メトリクス収集: データ不足")
            else:
                print(f"❌ メトリクス取得エラー: {response.status_code}")
                
        except Exception as e:
            print(f"❌ メトリクス分析例外: {e}")
    
    def step3_configuration_audit(self):
        """ステップ3: 設定監査"""
        print("⚙️ ステップ3: 設定監査")
        
        config_paths = [
            "/etc/datadog-agent/datadog.yaml",
            "/etc/datadog-agent/conf.d/"
        ]
        
        for config_path in config_paths:
            if subprocess.run(['test', '-r', config_path]).returncode == 0:
                print(f"✅ 設定ファイル読み取り可能: {config_path}")
            else:
                print(f"❌ 設定ファイル問題: {config_path}")
    
    def step4_performance_analysis(self):
        """ステップ4: パフォーマンス分析"""
        print("⚡ ステップ4: パフォーマンス診断")
        
        # Agent プロセス状況確認
        agent_ps = subprocess.run(['ps', 'aux'], capture_output=True, text=True)
        datadog_processes = [line for line in agent_ps.stdout.split('\n') 
                           if 'datadog-agent' in line]
        
        for process in datadog_processes:
            if process:
                parts = process.split()
                cpu_usage = parts[2]
                memory_usage = parts[3]
                print(f"   Agent CPU使用率: {cpu_usage}%")
                print(f"   Agent メモリ使用率: {memory_usage}%")
    
    def step5_comprehensive_report(self):
        """ステップ5: 包括レポート生成"""
        print("📋 ステップ5: 診断レポート生成")
        
        report = {
            "診断実行時刻": datetime.now().isoformat(),
            "Agent バージョン": self._get_agent_version(),
            "推奨アクション": self._generate_recommendations()
        }
        
        print(json.dumps(report, indent=2, ensure_ascii=False))
        
    def _get_agent_version(self):
        version_cmd = subprocess.run(['datadog-agent', 'version'], 
                                   capture_output=True, text=True)
        return version_cmd.stdout.strip() if version_cmd.returncode == 0 else "不明"
    
    def _generate_recommendations(self):
        return [
            "定期的なAgent再起動（月1回）",
            "設定ファイルのバックアップ作成",
            "ログレベルを一時的にDEBUGに変更して詳細調査",
            "Datadog サポートへの技術的情報提供準備"
        ]
    
    def run_complete_diagnosis(self):
        """完全診断実行"""
        print("🔍 Datadog 完全診断開始\n")
        self.step1_connectivity_check()
        self.step2_data_flow_analysis()
        self.step3_configuration_audit()
        self.step4_performance_analysis()
        self.step5_comprehensive_report()
        print("\n✅ 診断完了")

# 使用例
if __name__ == "__main__":
    troubleshooter = DatadogTroubleshooter("your-api-key", "your-app-key")
    troubleshooter.run_complete_diagnosis()

# メトリクス欠損自動分析ツール
import requests
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta

class MetricsGapAnalyzer:
    def __init__(self, api_key, app_key):
        self.api_key = api_key
        self.app_key = app_key
        self.base_url = "https://api.datadoghq.com/api/v1"
    
    def analyze_metrics_gap(self, metric_name, hours_back=24):
        """メトリクス欠損分析"""
        end_time = datetime.now()
        start_time = end_time - timedelta(hours=hours_back)
        
        try:
            response = requests.get(
                f"{self.base_url}/query",
                params={
                    "api_key": self.api_key,
                    "application_key": self.app_key,
                    "query": f"avg:{metric_name}{{*}}",
                    "from": int(start_time.timestamp()),
                    "to": int(end_time.timestamp())
                }
            )
            
            if response.status_code == 200:
                data = response.json()
                return self._detect_gaps(data)
            else:
                return f"エラー: {response.status_code}"
                
        except Exception as e:
            return f"例外: {e}"
    
    def _detect_gaps(self, data):
        """データギャップ検出"""
        if not data.get('series'):
            return "データ系列が見つかりません"
        
        series = data['series'][0]
        points = series.get('pointlist', [])
        
        if len(points) < 2:
            return "データポイントが不足しています"
        
        gaps = []
        expected_interval = 60  # 60秒間隔と仮定
        
        for i in range(1, len(points)):
            prev_time = points[i-1][0]
            curr_time = points[i][0]
            time_diff = (curr_time - prev_time) / 1000  # ミリ秒を秒に変換
            
            if time_diff > expected_interval * 2:  # 2倍以上の間隔
                gaps.append({
                    'start': datetime.fromtimestamp(prev_time/1000),
                    'end': datetime.fromtimestamp(curr_time/1000),
                    'duration_minutes': time_diff / 60
                })
        
        return {
            'total_points': len(points),
            'gaps_detected': len(gaps),
            'gaps': gaps,
            'data_completeness': f"{((len(points) - len(gaps)) / len(points) * 100):.1f}%"
        }
    
    def generate_gap_report(self, metrics_list):
        """複数メトリクスのギャップレポート生成"""
        report = {"分析実行時刻": datetime.now().isoformat(), "メトリクス分析": {}}
        
        for metric in metrics_list:
            report["メトリクス分析"][metric] = self.analyze_metrics_gap(metric)
        
        return report

# 使用例
analyzer = MetricsGapAnalyzer("your-api-key", "your-app-key")
important_metrics = [
    "system.cpu.user",
    "system.mem.used",
    "system.disk.used"
]

gap_report = analyzer.generate_gap_report(important_metrics)
print(json.dumps(gap_report, indent=2, ensure_ascii=False))

# 機械学習ベース異常検知設定自動化
import requests
import json
from datetime import datetime, timedelta

class DatadogMLAnomalyDetector:
    def __init__(self, api_key, app_key):
        self.api_key = api_key
        self.app_key = app_key
        self.base_url = "https://api.datadoghq.com/api/v1"
    
    def create_ml_anomaly_monitor(self, metric_name, sensitivity='medium'):
        """ML異常検知モニター作成"""
        
        monitor_config = {
            "type": "query alert",
            "query": f"avg(last_1h):anomalies(avg:{metric_name}{{*}}, 'basic', 2, direction='both', alert_window='last_15m', interval=60, count_default_zero='true') >= 1",
            "name": f"[ML] {metric_name} 異常検知",
            "message": f"""
🤖 機械学習による異常検知アラート

**メトリクス**: {metric_name}
**検知手法**: Basic Algorithm
**感度設定**: {sensitivity}
**検知時刻**: {{{{@timestamp}}}}

**異常パターン分析**:
- 過去データとの偏差: {{{{value}}}} 
- 信頼度レベル: {{{{confidence}}}}
- 推奨アクション: 詳細調査が必要

**関連ダッシュボード**: {{{{dashboard_link}}}}
**トラブルシューティングガイド**: {{{{runbook_link}}}}

@slack-alerts-channel
            """.strip(),
            "tags": ["ml-anomaly", "automated", f"sensitivity:{sensitivity}"],
            "options": {
                "notify_audit": False,
                "locked": False,
                "include_tags": True,
                "new_host_delay": 300,
                "require_full_window": True,
                "notify_no_data": False,
                "timeout_h": 24,
                "escalation_message": "異常状態が継続しています。エスカレーションが必要です。"
            }
        }
        
        try:
            response = requests.post(
                f"{self.base_url}/monitor",
                headers={
                    "Content-Type": "application/json",
                    "DD-API-KEY": self.api_key,
                    "DD-APPLICATION-KEY": self.app_key
                },
                json=monitor_config
            )
            
            if response.status_code == 200:
                monitor = response.json()
                return {
                    "status": "success",
                    "monitor_id": monitor['id'],
                    "monitor_url": f"https://app.datadoghq.com/monitors/{monitor['id']}"
                }
            else:
                return {"status": "error", "message": f"HTTP {response.status_code}"}
                
        except Exception as e:
            return {"status": "exception", "message": str(e)}
    
    def create_forecasting_monitor(self, metric_name, forecast_hours=24):
        """予測アラート作成"""
        
        forecast_config = {
            "type": "query alert",
            "query": f"avg(last_4h):forecast(avg:{metric_name}{{*}}, 'linear', 1, interval=60, history='1w', model='default', seasonality='auto') >= threshold",
            "name": f"[予測] {metric_name} 将来異常予測",
            "message": f"""
🔮 機械学習予測アラート

**メトリクス**: {metric_name}
**予測期間**: {forecast_hours}時間後
**予測手法**: Linear Regression + Seasonality
**信頼度**: {{{{confidence}}}}%

**予測結果**:
- 予測値: {{{{forecast_value}}}}
- 現在値: {{{{current_value}}}}
- 変化率: {{{{change_rate}}}}%

**推奨アクション**:
1. リソース準備・スケーリング計画
2. 関連チームへの事前通知
3. 予防的メンテナンス検討

@sre-team @capacity-planning
            """.strip(),
            "tags": ["forecasting", "predictive", "capacity-planning"],
            "options": {
                "notify_audit": False,
                "locked": False,
                "timeout_h": 48,
                "evaluation_delay": 900
            }
        }
        
        return self._create_monitor(forecast_config)
    
    def setup_ml_monitoring_suite(self, critical_metrics):
        """包括的ML監視スイート構築"""
        
        results = {
            "created_monitors": [],
            "failed_monitors": [],
            "summary": {}
        }
        
        for metric_name, config in critical_metrics.items():
            sensitivity = config.get('sensitivity', 'medium')
            enable_forecasting = config.get('forecasting', True)
            
            # 異常検知モニター作成
            anomaly_result = self.create_ml_anomaly_monitor(metric_name, sensitivity)
            if anomaly_result["status"] == "success":
                results["created_monitors"].append({
                    "type": "anomaly",
                    "metric": metric_name,
                    "monitor_id": anomaly_result["monitor_id"]
                })
            else:
                results["failed_monitors"].append({
                    "metric": metric_name,
                    "error": anomaly_result["message"]
                })
            
            # 予測モニター作成
            if enable_forecasting:
                forecast_result = self.create_forecasting_monitor(metric_name)
                if forecast_result["status"] == "success":
                    results["created_monitors"].append({
                        "type": "forecasting",
                        "metric": metric_name,
                        "monitor_id": forecast_result["monitor_id"]
                    })
        
        results["summary"] = {
            "total_created": len(results["created_monitors"]),
            "total_failed": len(results["failed_monitors"]),
            "success_rate": f"{(len(results['created_monitors']) / (len(results['created_monitors']) + len(results['failed_monitors'])) * 100):.1f}%"
        }
        
        return results
    
    def _create_monitor(self, config):
        """共通モニター作成メソッド"""
        try:
            response = requests.post(
                f"{self.base_url}/monitor",
                headers={
                    "Content-Type": "application/json",
                    "DD-API-KEY": self.api_key,
                    "DD-APPLICATION-KEY": self.app_key
                },
                json=config
            )
            
            if response.status_code == 200:
                monitor = response.json()
                return {
                    "status": "success",
                    "monitor_id": monitor['id']
                }
            else:
                return {"status": "error", "message": f"HTTP {response.status_code}"}
                
        except Exception as e:
            return {"status": "exception", "message": str(e)}

# 使用例: エンタープライズ規模のML監視実装
ml_detector = DatadogMLAnomalyDetector("your-api-key", "your-app-key")

# 重要メトリクスの定義
critical_business_metrics = {
    "aws.applicationelb.request_count": {
        "sensitivity": "high",
        "forecasting": True
    },
    "system.cpu.user": {
        "sensitivity": "medium", 
        "forecasting": True
    },
    "postgresql.database.commits": {
        "sensitivity": "medium",
        "forecasting": False
    },
    "custom.business.revenue_per_hour": {
        "sensitivity": "high",
        "forecasting": True
    }
}

# ML監視スイート構築実行
ml_suite_result = ml_detector.setup_ml_monitoring_suite(critical_business_metrics)
print(json.dumps(ml_suite_result, indent=2, ensure_ascii=False))

# エンタープライズビジネスメトリクス統合システム
import requests
import psycopg2
import redis
import json
from datetime import datetime
from datadog import initialize, api

class BusinessMetricsIntegrator:
    def __init__(self, datadog_config, db_config, redis_config):
        # Datadog初期化
        initialize(**datadog_config)
        
        # データベース接続
        self.db_conn = psycopg2.connect(**db_config)
        self.redis_client = redis.Redis(**redis_config)
    
    def collect_business_kpis(self):
        """ビジネスKPI収集・計算"""
        
        cursor = self.db_conn.cursor()
        
        # リアルタイム売上メトリクス
        cursor.execute("""
            SELECT 
                SUM(amount) as hourly_revenue,
                COUNT(*) as hourly_transactions,
                AVG(amount) as avg_transaction_value
            FROM orders 
            WHERE created_at >= NOW() - INTERVAL '1 hour'
        """)
        
        revenue_data = cursor.fetchone()
        
        # ユーザーエンゲージメント
        cursor.execute("""
            SELECT 
                COUNT(DISTINCT user_id) as active_users,
                AVG(session_duration) as avg_session_duration,
                SUM(page_views) as total_page_views
            FROM user_sessions 
            WHERE created_at >= NOW() - INTERVAL '1 hour'
        """)
        
        engagement_data = cursor.fetchone()
        
        # カスタマーサービスメトリクス
        cursor.execute("""
            SELECT 
                COUNT(*) as support_tickets_created,
                AVG(response_time_minutes) as avg_response_time,
                COUNT(CASE WHEN status = 'resolved' THEN 1 END) as tickets_resolved
            FROM support_tickets 
            WHERE created_at >= NOW() - INTERVAL '1 hour'
        """)
        
        support_data = cursor.fetchone()
        
        return {
            "revenue": {
                "hourly_revenue": float(revenue_data[0] or 0),
                "hourly_transactions": revenue_data[1],
                "avg_transaction_value": float(revenue_data[2] or 0)
            },
            "engagement": {
                "active_users": engagement_data[0],
                "avg_session_duration": float(engagement_data[1] or 0),
                "total_page_views": engagement_data[2]
            },
            "support": {
                "tickets_created": support_data[0],
                "avg_response_time": float(support_data[1] or 0),
                "tickets_resolved": support_data[2]
            }
        }
    
    def calculate_business_health_score(self, kpis):
        """ビジネス健康度スコア算出"""
        
        # キャッシュされた過去データ取得
        historical_data = self.redis_client.get("business_kpis_24h")
        if historical_data:
            historical_data = json.loads(historical_data)
        else:
            historical_data = kpis  # 初回実行時
        
        # 健康度スコア計算 (0-100)
        revenue_score = min((kpis["revenue"]["hourly_revenue"] / historical_data["revenue"].get("hourly_revenue", 1)) * 50, 50)
        engagement_score = min((kpis["engagement"]["active_users"] / historical_data["engagement"].get("active_users", 1)) * 30, 30)
        support_score = max(20 - (kpis["support"]["avg_response_time"] / 60), 0)  # 応答時間ベース
        
        total_score = revenue_score + engagement_score + support_score
        
        return {
            "total_score": round(total_score, 2),
            "revenue_component": round(revenue_score, 2),
            "engagement_component": round(engagement_score, 2),
            "support_component": round(support_score, 2),
            "health_status": self._get_health_status(total_score)
        }
    
    def _get_health_status(self, score):
        """健康度ステータス判定"""
        if score >= 80:
            return "excellent"
        elif score >= 60:
            return "good"
        elif score >= 40:
            return "warning"
        else:
            return "critical"
    
    def send_metrics_to_datadog(self, kpis, health_score):
        """Datadogへのメトリクス送信"""
        
        timestamp = datetime.now()
        
        # 収益メトリクス
        api.Metric.send(
            metric='business.revenue.hourly_total',
            points=[(timestamp, kpis["revenue"]["hourly_revenue"])],
            tags=['source:database', 'team:finance']
        )
        
        api.Metric.send(
            metric='business.revenue.transaction_count',
            points=[(timestamp, kpis["revenue"]["hourly_transactions"])],
            tags=['source:database', 'team:finance']
        )
        
        api.Metric.send(
            metric='business.revenue.avg_transaction_value',
            points=[(timestamp, kpis["revenue"]["avg_transaction_value"])],
            tags=['source:database', 'team:finance']
        )
        
        # エンゲージメントメトリクス
        api.Metric.send(
            metric='business.engagement.active_users',
            points=[(timestamp, kpis["engagement"]["active_users"])],
            tags=['source:database', 'team:product']
        )
        
        api.Metric.send(
            metric='business.engagement.avg_session_duration',
            points=[(timestamp, kpis["engagement"]["avg_session_duration"])],
            tags=['source:database', 'team:product']
        )
        
        # サポートメトリクス
        api.Metric.send(
            metric='business.support.tickets_created',
            points=[(timestamp, kpis["support"]["tickets_created"])],
            tags=['source:database', 'team:support']
        )
        
        api.Metric.send(
            metric='business.support.avg_response_time',
            points=[(timestamp, kpis["support"]["avg_response_time"])],
            tags=['source:database', 'team:support']
        )
        
        # ビジネス健康度スコア
        api.Metric.send(
            metric='business.health_score.total',
            points=[(timestamp, health_score["total_score"])],
            tags=['source:calculated', 'team:executive']
        )
        
        # 健康度ステータス（数値化）
        status_values = {"excellent": 4, "good": 3, "warning": 2, "critical": 1}
        api.Metric.send(
            metric='business.health_score.status',
            points=[(timestamp, status_values[health_score["health_status"]])],
            tags=['source:calculated', 'team:executive', f'status:{health_score["health_status"]}']
        )
        
        print(f"✅ ビジネスメトリクス送信完了: {timestamp}")
        print(f"   健康度スコア: {health_score['total_score']}/100 ({health_score['health_status']})")
    
    def run_collection_cycle(self):
        """メトリクス収集サイクル実行"""
        try:
            print("📊 ビジネスKPI収集開始...")
            
            # KPI収集
            kpis = self.collect_business_kpis()
            
            # 健康度スコア計算
            health_score = self.calculate_business_health_score(kpis)
            
            # Datadogに送信
            self.send_metrics_to_datadog(kpis, health_score)
            
            # Redisにキャッシュ保存（24時間TTL）
            self.redis_client.setex(
                "business_kpis_24h", 
                86400, 
                json.dumps(kpis)
            )
            
            print("✅ ビジネスメトリクス収集サイクル完了")
            
        except Exception as e:
            print(f"❌ エラー: {e}")
            # エラーメトリクス送信
            api.Metric.send(
                metric='business.metrics_collection.errors',
                points=[(datetime.now(), 1)],
                tags=['source:collector', 'error_type:collection_failure']
            )

# 設定例
datadog_config = {
    'api_key': 'your-datadog-api-key',
    'app_key': 'your-datadog-app-key'
}

db_config = {
    'host': 'your-db-host',
    'database': 'your-database',
    'user': 'your-username', 
    'password': 'your-password'
}

redis_config = {
    'host': 'your-redis-host',
    'port': 6379,
    'db': 0
}

# ビジネスメトリクス統合システム起動
integrator = BusinessMetricsIntegrator(datadog_config, db_config, redis_config)
integrator.run_collection_cycle()

# DevOpsパイプライン統合自動化システム
import requests
import json
import yaml
from datetime import datetime
from datadog import initialize, api, statsd

class DevOpsIntegrationFramework:
    def __init__(self, datadog_config):
        initialize(**datadog_config)
        self.statsd = statsd
        self.deployments = []
    
    def track_deployment_metrics(self, deployment_info):
        """デプロイメント追跡メトリクス"""
        
        # デプロイメント開始
        self.statsd.increment('deployment.started', 
                             tags=[
                                 f'service:{deployment_info["service"]}',
                                 f'version:{deployment_info["version"]}',
                                 f'environment:{deployment_info["environment"]}'
                             ])
        
        # デプロイメント期間追跡
        deployment_start = datetime.now()
        deployment_info['start_time'] = deployment_start
        
        return deployment_info
    
    def create_deployment_markers(self, deployment_info):
        """デプロイメントマーカー作成"""
        
        try:
            api.Event.create(
                title=f"🚀 Deployment: {deployment_info['service']} v{deployment_info['version']}",
                text=f"""
**デプロイメント詳細**:
- サービス: {deployment_info['service']}
- バージョン: {deployment_info['version']} 
- 環境: {deployment_info['environment']}
- デプロイ担当者: {deployment_info['deployer']}
- コミットハッシュ: {deployment_info.get('commit_hash', 'N/A')}

**デプロイメント設定**:
- 戦略: {deployment_info.get('strategy', 'Rolling Update')}
- レプリカ数: {deployment_info.get('replicas', 'N/A')}
- リソース制限: {deployment_info.get('resources', 'Default')}
                """.strip(),
                tags=[
                    'deployment',
                    f'service:{deployment_info["service"]}',
                    f'version:{deployment_info["version"]}',
                    f'environment:{deployment_info["environment"]}'
                ],
                alert_type='info'
            )
            
            print(f"✅ デプロイメントマーカー作成: {deployment_info['service']}")
            
        except Exception as e:
            print(f"❌ マーカー作成エラー: {e}")
    
    def monitor_deployment_health(self, deployment_info, health_check_duration=300):
        """デプロイメント後ヘルスチェック"""
        
        service = deployment_info['service']
        environment = deployment_info['environment']
        
        # ヘルスチェック用カスタムモニター作成
        health_monitor = {
            "type": "metric alert",
            "query": f"avg(last_5m):avg:trace.{service}.request.duration{{env:{environment}}} > 1000",
            "name": f"[Deploy Health] {service} Post-Deployment Performance",
            "message": f"""
🚨 デプロイメント後パフォーマンス異常

**サービス**: {service}
**環境**: {environment} 
**デプロイ時刻**: {deployment_info['start_time']}
**検知メトリクス**: レスポンス時間異常

**即座の対応**:
1. ロールバック検討
2. ログ詳細分析
3. リソース使用状況確認

@deployment-team @sre-team
            """.strip(),
            "tags": ["post-deployment", "health-check", f"service:{service}"],
            "options": {
                "timeout_h": 1,  # 1時間で自動無効化
                "require_full_window": False,
                "notify_no_data": True
            }
        }
        
        try:
            response = requests.post(
                "https://api.datadoghq.com/api/v1/monitor",
                headers={
                    "Content-Type": "application/json",
                    "DD-API-KEY": api._api_key,
                    "DD-APPLICATION-KEY": api._application_key
                },
                json=health_monitor
            )
            
            if response.status_code == 200:
                monitor_id = response.json()['id']
                print(f"✅ デプロイメントヘルスモニター作成: {monitor_id}")
                
                # 一定時間後にモニター削除をスケジュール
                self._schedule_monitor_cleanup(monitor_id, health_check_duration)
            
        except Exception as e:
            print(f"❌ ヘルスモニター作成エラー: {e}")
    
    def calculate_deployment_success_metrics(self, deployment_info):
        """デプロイメント成功メトリクス計算"""
        
        if 'start_time' in deployment_info:
            deployment_duration = (datetime.now() - deployment_info['start_time']).total_seconds()
            
            # デプロイメント期間メトリクス送信
            self.statsd.timing('deployment.duration', 
                              deployment_duration,
                              tags=[
                                  f'service:{deployment_info["service"]}',
                                  f'environment:{deployment_info["environment"]}'
                              ])
            
            # 成功率計算（簡略版）
            self.statsd.increment('deployment.completed',
                                 tags=[
                                     f'service:{deployment_info["service"]}',
                                     f'result:success'  # 実際は失敗判定ロジック必要
                                 ])
            
            print(f"📊 デプロイメント完了: {deployment_duration:.1f}秒")
    
    def _schedule_monitor_cleanup(self, monitor_id, duration):
        """モニタークリーンアップスケジュール"""
        # 実際の実装では、celery やcron job などを使用
        print(f"⏰ モニター {monitor_id} を {duration}秒後に削除予定")

# SREワークフロー統合例
devops_framework = DevOpsIntegrationFramework({
    'api_key': 'your-datadog-api-key',
    'app_key': 'your-datadog-app-key'
})

# デプロイメント情報
deployment = {
    'service': 'user-service',
    'version': 'v2.1.4',
    'environment': 'production',
    'deployer': 'sre-team',
    'commit_hash': 'abc123def456',
    'strategy': 'Blue-Green',
    'replicas': 5
}

# DevOpsパイプライン実行
tracked_deployment = devops_framework.track_deployment_metrics(deployment)
devops_framework.create_deployment_markers(tracked_deployment)
devops_framework.monitor_deployment_health(tracked_deployment)

# デプロイメント完了時
devops_framework.calculate_deployment_success_metrics(tracked_deployment)

# Datadog新機能追跡・評価システム
import requests
import json
from datetime import datetime, timedelta
import feedparser

class DatadogInnovationTracker:
    def __init__(self, datadog_config):
        self.api_key = datadog_config['api_key']
        self.app_key = datadog_config['app_key']
        self.feature_evaluation_queue = []
    
    def monitor_datadog_announcements(self):
        """Datadog公式発表の追跡"""
        
        sources = {
            "blog": "https://www.datadoghq.com/blog/feed/",
            "changelog": "https://docs.datadoghq.com/api/latest/",
            "github_releases": "https://api.github.com/repos/DataDog/datadog-agent/releases"
        }
        
        new_features = []
        
        try:
            # ブログフィード解析
            blog_feed = feedparser.parse(sources["blog"])
            for entry in blog_feed.entries[:5]:  # 最新5件
                if self._is_feature_announcement(entry.title):
                    new_features.append({
                        "type": "blog_post",
                        "title": entry.title,
                        "url": entry.link,
                        "published": entry.published,
                        "summary": entry.summary[:200] + "..."
                    })
            
            # GitHub Releases 確認
            github_response = requests.get(sources["github_releases"])
            if github_response.status_code == 200:
                releases = github_response.json()
                for release in releases[:3]:  # 最新3リリース
                    if self._is_significant_release(release):
                        new_features.append({
                            "type": "agent_release",
                            "version": release["tag_name"],
                            "url": release["html_url"],
                            "published": release["published_at"],
                            "notes": release["body"][:300] + "..."
                        })
            
            return new_features
            
        except Exception as e:
            print(f"❌ 発表追跡エラー: {e}")
            return []
    
    def _is_feature_announcement(self, title):
        """機能発表判定"""
        feature_keywords = [
            "introducing", "new feature", "now available", "announcing",
            "machine learning", "AI", "automation", "integration"
        ]
        title_lower = title.lower()
        return any(keyword in title_lower for keyword in feature_keywords)
    
    def _is_significant_release(self, release):
        """重要リリース判定"""
        version = release.get("tag_name", "")
        body = release.get("body", "").lower()
        
        # メジャーバージョンまたは重要な機能追加
        return ("7." in version and not release.get("prerelease", False)) or \
               any(keyword in body for keyword in ["feature", "enhancement", "improvement"])
    
    def evaluate_feature_impact(self, feature):
        """機能の影響評価"""
        
        evaluation_criteria = {
            "business_impact": 0,      # ビジネス価値 (1-10)
            "implementation_effort": 0,  # 実装工数 (1-10) 
            "technical_complexity": 0,   # 技術的複雑さ (1-10)
            "urgency": 0,              # 緊急性 (1-10)
            "strategic_alignment": 0    # 戦略適合性 (1-10)
        }
        
        # 簡略化された自動評価（実際はより詳細な分析が必要）
        feature_text = f"{feature.get('title', '')} {feature.get('summary', '')}".lower()
        
        # ビジネス価値評価
        if any(keyword in feature_text for keyword in ["revenue", "cost", "efficiency", "automation"]):
            evaluation_criteria["business_impact"] = 8
        elif any(keyword in feature_text for keyword in ["monitoring", "alerting", "dashboard"]):
            evaluation_criteria["business_impact"] = 6
        else:
            evaluation_criteria["business_impact"] = 4
        
        # 実装工数評価
        if "machine learning" in feature_text or "ai" in feature_text:
            evaluation_criteria["implementation_effort"] = 8
        elif "integration" in feature_text:
            evaluation_criteria["implementation_effort"] = 6
        else:
            evaluation_criteria["implementation_effort"] = 4
        
        # 緊急性評価
        if "security" in feature_text or "compliance" in feature_text:
            evaluation_criteria["urgency"] = 9
        elif "performance" in feature_text:
            evaluation_criteria["urgency"] = 7
        else:
            evaluation_criteria["urgency"] = 5
        
        # 総合スコア計算
        total_score = sum(evaluation_criteria.values())
        priority = "High" if total_score >= 35 else "Medium" if total_score >= 25 else "Low"
        
        return {
            "feature": feature,
            "evaluation": evaluation_criteria,
            "total_score": total_score,
            "priority": priority,
            "recommendation": self._generate_recommendation(evaluation_criteria, priority)
        }
    
    def _generate_recommendation(self, criteria, priority):
        """推奨アクション生成"""
        if priority == "High":
            return {
                "action": "immediate_evaluation",
                "timeline": "1-2 weeks",
                "next_steps": [
                    "詳細技術調査の実施",
                    "パイロットプロジェクト計画",
                    "リソース配分検討",
                    "ステークホルダー承認取得"
                ]
            }
        elif priority == "Medium":
            return {
                "action": "planned_evaluation",
                "timeline": "1-2 months",
                "next_steps": [
                    "四半期計画への組み込み",
                    "技術検証スケジューリング",
                    "ROI分析実施"
                ]
            }
        else:
            return {
                "action": "monitoring",
                "timeline": "継続監視",
                "next_steps": [
                    "定期レビューでの再評価",
                    "コミュニティフィードバック監視"
                ]
            }
    
    def create_innovation_roadmap(self, evaluated_features):
        """イノベーションロードマップ作成"""
        
        high_priority = [f for f in evaluated_features if f["priority"] == "High"]
        medium_priority = [f for f in evaluated_features if f["priority"] == "Medium"]
        
        roadmap = {
            "generated_date": datetime.now().isoformat(),
            "summary": {
                "total_features": len(evaluated_features),
                "high_priority": len(high_priority),
                "medium_priority": len(medium_priority),
                "low_priority": len(evaluated_features) - len(high_priority) - len(medium_priority)
            },
            "quarterly_plan": {
                "Q1": [f["feature"]["title"] for f in high_priority[:2]],
                "Q2": [f["feature"]["title"] for f in high_priority[2:4]] + [f["feature"]["title"] for f in medium_priority[:1]],
                "Q3": [f["feature"]["title"] for f in medium_priority[1:3]],
                "Q4": [f["feature"]["title"] for f in medium_priority[3:5]]
            },
            "resource_requirements": self._calculate_resource_requirements(evaluated_features)
        }
        
        return roadmap
    
    def _calculate_resource_requirements(self, features):
        """リソース要件計算"""
        high_priority_features = [f for f in features if f["priority"] == "High"]
        
        total_effort = sum(f["evaluation"]["implementation_effort"] for f in high_priority_features)
        complexity_score = sum(f["evaluation"]["technical_complexity"] for f in high_priority_features)
        
        return {
            "estimated_effort_weeks": total_effort * 2,  # 簡略計算
            "required_expertise": ["Datadog Platform", "Machine Learning", "DevOps"],
            "budget_estimate": f"${total_effort * 5000}-${total_effort * 8000}",  # 概算
            "team_size": max(2, min(6, total_effort // 5))
        }

# 使用例: 継続的イノベーション管理
innovation_tracker = DatadogInnovationTracker({
    'api_key': 'your-datadog-api-key',
    'app_key': 'your-datadog-app-key'
})

# 新機能監視実行
new_features = innovation_tracker.monitor_datadog_announcements()
print(f"📡 発見された新機能: {len(new_features)}件")

# 各機能の評価
evaluated_features = []
for feature in new_features:
    evaluation = innovation_tracker.evaluate_feature_impact(feature)
    evaluated_features.append(evaluation)
    print(f"🔍 評価完了: {feature['title']} - {evaluation['priority']} Priority")

# イノベーションロードマップ生成
roadmap = innovation_tracker.create_innovation_roadmap(evaluated_features)
print(json.dumps(roadmap, indent=2, ensure_ascii=False))