diff --git a/core-web/apps/dotcms-ui/src/app/app.routes.ts b/core-web/apps/dotcms-ui/src/app/app.routes.ts index 2ba39163544b..8e87a2de842f 100644 --- a/core-web/apps/dotcms-ui/src/app/app.routes.ts +++ b/core-web/apps/dotcms-ui/src/app/app.routes.ts @@ -22,6 +22,12 @@ import { MainCoreLegacyComponent } from './view/components/main-core-legacy/main import { MainComponentLegacyComponent } from './view/components/main-legacy/main-legacy.component'; const PORTLETS_ANGULAR: Route[] = [ + { + path: 'dotCDN', + canActivate: [MenuGuardService], + canActivateChild: [MenuGuardService], + loadChildren: () => import('@dotcms/portlets/dot-cdn/portlet').then((m) => m.dotCdnRoutes) + }, { path: 'containers', loadChildren: () => diff --git a/core-web/libs/portlets/dot-cdn/.eslintrc.json b/core-web/libs/portlets/dot-cdn/.eslintrc.json new file mode 100644 index 000000000000..ef536cdfaf37 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../../.eslintrc.base.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/core-web/libs/portlets/dot-cdn/project.json b/core-web/libs/portlets/dot-cdn/project.json new file mode 100644 index 000000000000..19d8a93794bd --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/project.json @@ -0,0 +1,13 @@ +{ + "name": "portlets-dot-cdn-portlet", + "$schema": "../../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "libs/portlets/dot-cdn/src", + "prefix": "dot", + "projectType": "library", + "tags": ["type:feature", "scope:dotcms-ui", "portlet:cdn"], + "targets": { + "lint": { + "executor": "@nx/eslint:lint" + } + } +} diff --git a/core-web/libs/portlets/dot-cdn/src/index.ts b/core-web/libs/portlets/dot-cdn/src/index.ts new file mode 100644 index 000000000000..44c9365302f3 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/index.ts @@ -0,0 +1 @@ +export * from './lib/lib.routes'; diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn-filters/dot-cdn-filters.component.ts b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn-filters/dot-cdn-filters.component.ts new file mode 100644 index 000000000000..dcf9b7ffaf2b --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn-filters/dot-cdn-filters.component.ts @@ -0,0 +1,165 @@ +import { format, isBefore, startOfDay, subDays } from 'date-fns'; + +import { ChangeDetectionStrategy, Component, computed, model, output, signal } from '@angular/core'; +import { FormsModule } from '@angular/forms'; + +import { ButtonModule } from 'primeng/button'; +import { DatePickerModule } from 'primeng/datepicker'; +import { SelectButtonModule, SelectButtonChangeEvent } from 'primeng/selectbutton'; + +export interface CdnDateFilter { + dateFrom: string; + dateTo: string; + hourly: boolean; +} + +export interface CdnFilterOption { + label: string; + value: string; +} + +export const CDN_TIME_PRESETS: CdnFilterOption[] = [ + { label: 'Today', value: 'today' }, + { label: '24h', value: 'last24h' }, + { label: '7d', value: 'last7d' }, + { label: '30d', value: 'last30d' }, + { label: '90d', value: 'last90d' }, + { label: 'Custom', value: 'custom' } +]; + +@Component({ + selector: 'dot-cdn-filters', + standalone: true, + imports: [FormsModule, SelectButtonModule, DatePickerModule, ButtonModule], + template: ` +
+ + + @if ($showDatePicker()) { + + + + + + } +
+ `, + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class DotCdnFiltersComponent { + readonly presets = CDN_TIME_PRESETS; + readonly $today = signal(startOfDay(new Date())); + readonly $selectedPreset = model('last30d'); + readonly $customRange = model(null); + readonly $rangeStart = signal(null); + + readonly $showDatePicker = computed(() => this.$selectedPreset() === 'custom'); + + filterChange = output(); + + onPresetChange(event: SelectButtonChangeEvent): void { + const preset = event.value as string; + if (preset === 'custom') { + return; + } + + this.$customRange.set(null); + this.$rangeStart.set(null); + this.filterChange.emit(this.resolvePreset(preset)); + } + + onDateSelect(date: Date): void { + const start = this.$rangeStart(); + + if (start === null || isBefore(date, start)) { + this.$rangeStart.set(startOfDay(date)); + } else { + this.$rangeStart.set(null); + const dateFrom = format(start, 'yyyy-MM-dd'); + const dateTo = format(date, 'yyyy-MM-dd'); + const diffDays = Math.ceil((date.getTime() - start.getTime()) / (1000 * 60 * 60 * 24)); + + this.filterChange.emit({ + dateFrom, + dateTo, + hourly: diffDays <= 2 + }); + } + } + + clearDateRange(): void { + this.$rangeStart.set(null); + this.$customRange.set(null); + } + + onCalendarClosed(): void { + const range = this.$customRange(); + if (!range || range.length !== 2) { + this.$rangeStart.set(null); + } + } + + private resolvePreset(preset: string): CdnDateFilter { + const today = format(new Date(), 'yyyy-MM-dd'); + + switch (preset) { + case 'today': + return { dateFrom: today, dateTo: today, hourly: true }; + case 'last24h': + return { + dateFrom: format(subDays(new Date(), 1), 'yyyy-MM-dd'), + dateTo: today, + hourly: true + }; + case 'last7d': + return { + dateFrom: format(subDays(new Date(), 7), 'yyyy-MM-dd'), + dateTo: today, + hourly: false + }; + case 'last30d': + return { + dateFrom: format(subDays(new Date(), 30), 'yyyy-MM-dd'), + dateTo: today, + hourly: false + }; + case 'last90d': + return { + dateFrom: format(subDays(new Date(), 90), 'yyyy-MM-dd'), + dateTo: today, + hourly: false + }; + default: + return { + dateFrom: format(subDays(new Date(), 30), 'yyyy-MM-dd'), + dateTo: today, + hourly: false + }; + } + } +} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.html b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.html new file mode 100644 index 000000000000..5de655f32235 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.html @@ -0,0 +1,183 @@ +@if (vm$ | async; as VM) { +
+ + + Overview + Flush Cache + + + + +
+ + @if (VM.cdnDomain) { + + CDN: + {{ VM.cdnDomain }} + + } +
+ +
+ +
+ @for (stats of VM.statsData; track stats) { +
+ +
+ + {{ stats.label }} + + @if (VM.isChartLoading) { + + } @else { +
+ {{ stats.value }} +
+ } +
+
+ } +
+ + +
+
+ @if (VM.isChartLoading) { + + } @else { +

+ Bandwidth Served +

+ + } +
+ +
+ @if (VM.isChartLoading) { + + } @else { +

+ Requests Served +

+ + } +
+ +
+ @if (VM.isChartLoading) { + + } @else { +

+ Cache Hit Rate +

+ + } +
+ +
+ @if (VM.isChartLoading) { + + } @else { +

Errors

+ + } +
+
+
+ + + @if (vmPurgeLoaders$ | async; as VMPurgeLoaders) { + +
+
+

Purge URL List

+

+ Purging removes files from CDN cache and re-downloads from your + origin. Enter exact CDN URLs, one per line. Use + * + for wildcards. +

+
+ + +
+
+ +
+

+ Purge Entire Cache +

+

+ Forces everything to be re-downloaded from your origin. May + cause a traffic spike if your CDN domain is heavily trafficked. +

+ +
+
+ + } + + +
+} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.scss b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.scss new file mode 100644 index 000000000000..2085f65e2bf7 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.scss @@ -0,0 +1,14 @@ +p-skeleton { + display: flex; + flex-direction: column; + justify-content: center; +} + +dot-spinner { + margin: 0 auto; + + width: max-content; + display: flex; + align-items: center; + justify-content: center; +} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.ts b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.ts new file mode 100644 index 000000000000..2e16967adcc3 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.component.ts @@ -0,0 +1,207 @@ +import { ChartOptions } from 'chart.js'; +import { Observable } from 'rxjs'; + +import { AsyncPipe, NgStyle } from '@angular/common'; +import { Component, inject, OnInit } from '@angular/core'; +import { + FormsModule, + ReactiveFormsModule, + UntypedFormBuilder, + UntypedFormGroup +} from '@angular/forms'; + +import { ButtonModule } from 'primeng/button'; +import { ChartModule } from 'primeng/chart'; +import { SkeletonModule } from 'primeng/skeleton'; +import { TabsModule } from 'primeng/tabs'; +import { TextareaModule } from 'primeng/textarea'; + +import { take } from 'rxjs/operators'; + +import { DotIconComponent, DotSpinnerComponent } from '@dotcms/ui'; + +import { CdnDateFilter, DotCdnFiltersComponent } from './dot-cdn-filters/dot-cdn-filters.component'; +import { CdnChartOptions, DotCDNState } from './dot-cdn.models'; +import { DotCDNStore } from './dot-cdn.store'; + +@Component({ + selector: 'dot-cdn', + templateUrl: './dot-cdn.component.html', + styleUrls: ['./dot-cdn.component.scss'], + standalone: true, + imports: [ + AsyncPipe, + NgStyle, + FormsModule, + ReactiveFormsModule, + TabsModule, + ChartModule, + ButtonModule, + TextareaModule, + SkeletonModule, + DotIconComponent, + DotSpinnerComponent, + DotCdnFiltersComponent + ], + providers: [DotCDNStore] +}) +export class DotCDNComponent implements OnInit { + private fb = inject(UntypedFormBuilder); + private dotCdnStore = inject(DotCDNStore); + + purgeZoneForm: UntypedFormGroup; + vm$: Observable> = + this.dotCdnStore.vm$; + vmPurgeLoaders$: Observable> = + this.dotCdnStore.vmPurgeLoaders$; + chartHeight = '25rem'; + options: CdnChartOptions; + + ngOnInit(): void { + this.setChartOptions(); + this.purgeZoneForm = this.fb.group({ + purgeUrlsTextArea: '' + }); + } + + onFilterChange(filter: CdnDateFilter): void { + this.dotCdnStore.loadStats(filter); + } + + purgePullZone(): void { + this.dotCdnStore.purgeCDNCacheAll(); + } + + purgeUrls(): void { + const urls: string[] = this.purgeZoneForm + .get('purgeUrlsTextArea') + .value.split('\n') + .map((url: string) => url.trim()); + + this.dotCdnStore + .purgeCDNCache(urls) + .pipe(take(1)) + .subscribe(() => { + this.purgeZoneForm.setValue({ purgeUrlsTextArea: '' }); + }); + } + + private setChartOptions(): void { + const bandwidthOptions: ChartOptions = { + responsive: true, + plugins: { + tooltip: { + callbacks: { + label: function (context) { + return `${context.dataset.label}: ${context.formattedValue}`; + } + } + } + }, + scales: { + x: { + ticks: { + maxTicksLimit: 15, + maxRotation: 45, + minRotation: 0 + } + }, + y: { + beginAtZero: true, + ticks: { + callback: function (value: number): string { + return value.toLocaleString('en-US'); + } + } + } + } + }; + + const requestOptions: ChartOptions = { + responsive: true, + plugins: { + tooltip: { + callbacks: { + label: function (context) { + return `${context.dataset.label}: ${Number(context.raw).toLocaleString('en-US')}`; + } + } + } + }, + scales: { + x: { + ticks: { + maxTicksLimit: 15, + maxRotation: 45, + minRotation: 0 + } + }, + y: { + beginAtZero: true, + ticks: { + precision: 0, + callback: function (value: number): string { + return value.toLocaleString('en-US'); + } + } + } + } + }; + + const commonScales = { + x: { ticks: { maxTicksLimit: 10, maxRotation: 45, minRotation: 0 } }, + y: { beginAtZero: true } + }; + + const cacheHitRateOptions: ChartOptions = { + responsive: true, + plugins: { + tooltip: { + callbacks: { + label: (context) => + `${context.dataset.label}: ${Number(context.raw).toFixed(2)}%` + } + } + }, + scales: { + ...commonScales, + y: { + beginAtZero: true, + max: 100, + ticks: { + callback: (value: number): string => `${value}%` + } + } + } + }; + + const errorOptions: ChartOptions = { + responsive: true, + plugins: { + tooltip: { + callbacks: { + label: (context) => + `${context.dataset.label}: ${Number(context.raw).toLocaleString('en-US')}` + } + } + }, + scales: { + ...commonScales, + y: { + beginAtZero: true, + ticks: { + precision: 0, + callback: (value: number): string => value.toLocaleString('en-US') + } + } + } + }; + + this.options = { + bandwidthUsedChart: bandwidthOptions, + requestsServedChart: requestOptions, + cacheHitRateChart: cacheHitRateOptions, + errorChart: errorOptions + }; + } +} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.models.ts b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.models.ts new file mode 100644 index 000000000000..b857aa4efae1 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.models.ts @@ -0,0 +1,81 @@ +import { ChartOptions } from 'chart.js'; + +export interface ChartDataSet { + label: string; + data: string[]; + borderColor?: string; + fill?: boolean; +} + +export interface DotCDNStats { + stats: { + bandwidthPretty: string; + bandwidthUsedChart: { [key: string]: number }; + requestsServedChart: { [key: string]: number }; + cacheHitRateChart: { [key: string]: number }; + originResponseTimeChart: { [key: string]: number }; + error4xxChart: { [key: string]: number }; + error5xxChart: { [key: string]: number }; + cacheHitRate: number; + averageOriginResponseTime: number; + dateFrom: string; + dateTo: string; + geographicDistribution: unknown; + totalBandwidthUsed: number; + totalRequestsServed: number; + cdnDomain: string; + }; +} + +export interface ChartData { + labels: string[]; + datasets: ChartDataSet[]; +} + +export interface DotChartStats { + label: string; + value: string; + icon: string; +} + +export interface PurgeUrlOptions { + hostId: string; + invalidateAll: boolean; + urls?: string[]; +} + +export interface DotCDNState { + chartBandwidthData: ChartData; + chartRequestsData: ChartData; + chartCacheHitRateData: ChartData; + chartErrorData: ChartData; + statsData: DotChartStats[]; + isChartLoading: boolean; + cdnDomain: string; + isPurgeUrlsLoading: boolean; + isPurgeZoneLoading: boolean; +} + +export type CdnChartOptions = { + bandwidthUsedChart: ChartOptions; + requestsServedChart: ChartOptions; + cacheHitRateChart: ChartOptions; + errorChart: ChartOptions; +}; + +export interface PurgeReturnData { + success: boolean; + invalidateAll: boolean; +} + +export const enum LoadingState { + IDLE = 'IDLE', + LOADING = 'LOADING', + LOADED = 'LOADED' +} + +export const enum Loader { + CHART = 'CHART', + PURGE_URLS = 'PURGE_URLS', + PURGE_PULL_ZONE = 'PURGE_PULL_ZONE' +} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.service.ts b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.service.ts new file mode 100644 index 000000000000..14a89ed49474 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.service.ts @@ -0,0 +1,76 @@ +import { Observable } from 'rxjs'; + +import { HttpClient } from '@angular/common/http'; +import { inject, Injectable } from '@angular/core'; + +import { map, mergeMap } from 'rxjs/operators'; + +import { SiteService } from '@dotcms/dotcms-js'; +import { DotCMSResponse } from '@dotcms/dotcms-models'; + +import { DotCDNStats, PurgeReturnData, PurgeUrlOptions } from './dot-cdn.models'; + +export interface StatsRequest { + dateFrom: string; + dateTo: string; + hourly?: boolean; +} + +@Injectable({ + providedIn: 'root' +}) +export class DotCDNService { + private http = inject(HttpClient); + private siteService = inject(SiteService); + + requestStats(request: StatsRequest): Observable { + return this.siteService.getCurrentSite().pipe( + map((site) => site.identifier), + mergeMap((hostId: string) => { + let url = + `/api/v1/dotcdn/stats?hostId=${hostId}` + + `&dateFrom=${request.dateFrom}&dateTo=${request.dateTo}`; + + if (request.hourly) { + url += '&hourly=true'; + } + + return this.http + .get>(url) + .pipe(map((response) => response.entity)); + }) + ); + } + + purgeCache(urls?: string[]): Observable { + return this.siteService.getCurrentSite().pipe( + map((site) => site.identifier), + mergeMap((hostId: string) => { + return this.purgeUrlRequest({ hostId, invalidateAll: false, urls }); + }), + map((response) => response.entity) + ); + } + + purgeCacheAll(): Observable { + return this.siteService.getCurrentSite().pipe( + map((site) => site.identifier), + mergeMap((hostId: string) => this.purgeUrlRequest({ hostId, invalidateAll: true })), + map((response) => response.entity) + ); + } + + private purgeUrlRequest({ + urls = [], + invalidateAll, + hostId + }: PurgeUrlOptions): Observable> { + return this.http.request>('DELETE', '/api/v1/dotcdn', { + body: { + urls, + invalidateAll, + hostId + } + }); + } +} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.store.ts b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.store.ts new file mode 100644 index 000000000000..d574906dd483 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/dot-cdn.store.ts @@ -0,0 +1,341 @@ +import { ComponentStore } from '@ngrx/component-store'; +import { tapResponse } from '@ngrx/operators'; +import { format, subDays } from 'date-fns'; +import { Observable, of } from 'rxjs'; + +import { HttpErrorResponse } from '@angular/common/http'; +import { inject, Injectable } from '@angular/core'; + +import { finalize, mergeMap, switchMap } from 'rxjs/operators'; + +import { DotHttpErrorManagerService } from '@dotcms/data-access'; + +import { CdnDateFilter } from './dot-cdn-filters/dot-cdn-filters.component'; +import { + ChartData, + DotCDNState, + DotCDNStats, + DotChartStats, + Loader, + LoadingState, + PurgeReturnData +} from './dot-cdn.models'; +import { DotCDNService, StatsRequest } from './dot-cdn.service'; + +const DEFAULT_FILTER: CdnDateFilter = { + dateFrom: format(subDays(new Date(), 30), 'yyyy-MM-dd'), + dateTo: format(new Date(), 'yyyy-MM-dd'), + hourly: false +}; + +@Injectable() +export class DotCDNStore extends ComponentStore { + private readonly dotCdnService = inject(DotCDNService); + private readonly httpErrorManager = inject(DotHttpErrorManagerService); + + constructor() { + super({ + chartBandwidthData: { labels: [], datasets: [] }, + chartRequestsData: { labels: [], datasets: [] }, + chartCacheHitRateData: { labels: [], datasets: [] }, + chartErrorData: { labels: [], datasets: [] }, + cdnDomain: '', + statsData: [], + isChartLoading: false, + isPurgeUrlsLoading: false, + isPurgeZoneLoading: false + }); + this.loadStats(DEFAULT_FILTER); + } + + readonly vm$ = this.select( + ({ + isChartLoading, + chartBandwidthData, + chartRequestsData, + chartCacheHitRateData, + chartErrorData, + statsData, + cdnDomain + }) => ({ + chartBandwidthData, + chartRequestsData, + chartCacheHitRateData, + chartErrorData, + statsData, + isChartLoading, + cdnDomain + }) + ); + + readonly vmPurgeLoaders$ = this.select(({ isPurgeUrlsLoading, isPurgeZoneLoading }) => ({ + isPurgeUrlsLoading, + isPurgeZoneLoading + })); + + readonly updateChartState = this.updater( + ( + state, + chartData: Omit< + DotCDNState, + 'isChartLoading' | 'isPurgeUrlsLoading' | 'isPurgeZoneLoading' + > + ) => ({ + ...state, + chartBandwidthData: chartData.chartBandwidthData, + chartRequestsData: chartData.chartRequestsData, + chartCacheHitRateData: chartData.chartCacheHitRateData, + chartErrorData: chartData.chartErrorData, + cdnDomain: chartData.cdnDomain, + statsData: chartData.statsData + }) + ); + + loadStats = this.effect((filter$: Observable): Observable => { + return filter$.pipe( + mergeMap((filter: CdnDateFilter) => { + this.dispatchLoading({ + loadingState: LoadingState.LOADING, + loader: Loader.CHART + }); + + const request: StatsRequest = { + dateFrom: filter.dateFrom, + dateTo: filter.dateTo, + hourly: filter.hourly + }; + + return this.dotCdnService.requestStats(request).pipe( + tapResponse({ + next: (data: DotCDNStats) => { + const result = this.getChartStatsData(data, filter.hourly); + this.updateChartState(result); + }, + error: (error: HttpErrorResponse) => this.httpErrorManager.handle(error) + }), + finalize(() => + this.dispatchLoading({ + loadingState: LoadingState.LOADED, + loader: Loader.CHART + }) + ) + ); + }) + ); + }); + + readonly dispatchLoading = this.updater( + (state, action: { loadingState: string; loader: string }): DotCDNState => { + switch (action.loader) { + case Loader.CHART: + return { + ...state, + isChartLoading: action.loadingState === LoadingState.LOADING + }; + case Loader.PURGE_URLS: + return { + ...state, + isPurgeUrlsLoading: action.loadingState === LoadingState.LOADING + }; + case Loader.PURGE_PULL_ZONE: + return { + ...state, + isPurgeZoneLoading: action.loadingState === LoadingState.LOADING + }; + default: + return state; + } + } + ); + + purgeCDNCache(urls: string[]): Observable { + const loading$ = of( + this.dispatchLoading({ + loadingState: LoadingState.LOADING, + loader: Loader.PURGE_URLS + }) + ); + + return loading$.pipe( + switchMap(() => + this.dotCdnService.purgeCache(urls).pipe( + finalize(() => + this.dispatchLoading({ + loadingState: LoadingState.LOADED, + loader: Loader.PURGE_URLS + }) + ) + ) + ) + ); + } + + readonly purgeCDNCacheAll = this.effect((trigger$: Observable) => { + return trigger$.pipe( + switchMap(() => { + this.dispatchLoading({ + loadingState: LoadingState.LOADING, + loader: Loader.PURGE_PULL_ZONE + }); + + return this.dotCdnService.purgeCacheAll().pipe( + tapResponse({ + next: () => undefined, + error: (error: HttpErrorResponse) => this.httpErrorManager.handle(error) + }), + finalize(() => + this.dispatchLoading({ + loadingState: LoadingState.LOADED, + loader: Loader.PURGE_PULL_ZONE + }) + ) + ); + }) + ); + }); + + private getChartStatsData({ stats }: DotCDNStats, hourly: boolean) { + const bandwidthValues = Object.values(stats.bandwidthUsedChart); + const { divisor, unit } = DotCDNStore.pickBandwidthUnit(bandwidthValues); + const labelFormatter = hourly ? DotCDNStore.formatHourLabel : DotCDNStore.formatDayLabel; + + const chartBandwidthData: ChartData = { + labels: Object.keys(stats.bandwidthUsedChart).map(labelFormatter), + datasets: [ + { + label: `Bandwidth (${unit})`, + data: bandwidthValues.map((v) => (v / divisor).toFixed(2).toString()), + borderColor: '#6f5fa3', + fill: false + } + ] + }; + + const chartRequestsData: ChartData = { + labels: Object.keys(stats.requestsServedChart).map(labelFormatter), + datasets: [ + { + label: 'Requests Served', + data: Object.values(stats.requestsServedChart).map((value: number): string => + value.toString() + ), + borderColor: '#FFA726', + fill: false + } + ] + }; + + const cacheHitKeys = Object.keys(stats.cacheHitRateChart || {}); + const chartCacheHitRateData: ChartData = { + labels: cacheHitKeys.map(labelFormatter), + datasets: [ + { + label: 'Cache Hit Rate (%)', + data: Object.values(stats.cacheHitRateChart || {}).map((v: number): string => + v.toFixed(2) + ), + borderColor: '#1ea97c', + fill: false + } + ] + }; + + const errorKeys = Object.keys(stats.error4xxChart || {}); + const chartErrorData: ChartData = { + labels: errorKeys.map(labelFormatter), + datasets: [ + { + label: '4xx Errors', + data: Object.values(stats.error4xxChart || {}).map((v: number): string => + v.toString() + ), + borderColor: '#FFA726', + fill: false + }, + { + label: '5xx Errors', + data: Object.values(stats.error5xxChart || {}).map((v: number): string => + v.toString() + ), + borderColor: '#f65446', + fill: false + } + ] + }; + + const statsData: DotChartStats[] = [ + { + label: 'Bandwidth Used', + value: stats.bandwidthPretty, + icon: 'insert_chart_outlined' + }, + { + label: 'Requests Served', + value: DotCDNStore.formatNumber(stats.totalRequestsServed), + icon: 'file_download' + }, + { + label: 'Cache Hit Rate', + value: `${stats.cacheHitRate.toFixed(2)}%`, + icon: 'speed' + }, + { + label: 'Avg Origin Response', + value: + stats.averageOriginResponseTime != null + ? `${stats.averageOriginResponseTime}ms` + : 'N/A', + icon: 'timer' + } + ]; + + return { + chartBandwidthData, + chartRequestsData, + chartCacheHitRateData, + chartErrorData, + statsData, + cdnDomain: stats.cdnDomain + }; + } + + private static pickBandwidthUnit(values: number[]): { divisor: number; unit: string } { + const max = values.reduce((a, b) => Math.max(a, b), 0); + if (max >= 1e9) { + return { divisor: 1e9, unit: 'GB' }; + } else if (max >= 1e6) { + return { divisor: 1e6, unit: 'MB' }; + } else if (max >= 1e3) { + return { divisor: 1e3, unit: 'KB' }; + } + + return { divisor: 1, unit: 'B' }; + } + + private static formatNumber(value: number): string { + return value.toLocaleString('en-US'); + } + + /** + * Format for hourly data: "Apr 1 2pm" + */ + private static formatHourLabel(key: string): string { + const date = new Date(key); + + return ( + date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }) + + ' ' + + date.toLocaleTimeString('en-US', { hour: 'numeric', hour12: true }) + ); + } + + /** + * Format for daily data: "Apr 1" + */ + private static formatDayLabel(key: string): string { + return new Date(key).toLocaleDateString('en-US', { + month: 'short', + day: 'numeric' + }); + } +} diff --git a/core-web/libs/portlets/dot-cdn/src/lib/lib.routes.ts b/core-web/libs/portlets/dot-cdn/src/lib/lib.routes.ts new file mode 100644 index 000000000000..bb5d80961c2e --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/src/lib/lib.routes.ts @@ -0,0 +1,10 @@ +import { Route } from '@angular/router'; + +import { DotCDNComponent } from './dot-cdn.component'; + +export const dotCdnRoutes: Route[] = [ + { + path: '', + component: DotCDNComponent + } +]; diff --git a/core-web/libs/portlets/dot-cdn/tsconfig.json b/core-web/libs/portlets/dot-cdn/tsconfig.json new file mode 100644 index 000000000000..8a932ccbe168 --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "target": "es2022", + "forceConsistentCasingInFileNames": true, + "noFallthroughCasesInSwitch": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ], + "angularCompilerOptions": { + "enableI18nLegacyMessageIdFormat": false, + "strictInjectionParameters": true, + "strictInputAccessModifiers": true, + "strictTemplates": true + } +} diff --git a/core-web/libs/portlets/dot-cdn/tsconfig.lib.json b/core-web/libs/portlets/dot-cdn/tsconfig.lib.json new file mode 100644 index 000000000000..d62a8082de0b --- /dev/null +++ b/core-web/libs/portlets/dot-cdn/tsconfig.lib.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../../dist/out-tsc", + "declaration": true, + "declarationMap": true, + "inlineSources": true, + "types": [] + }, + "exclude": ["src/**/*.spec.ts", "src/test-setup.ts", "jest.config.ts", "src/**/*.test.ts"], + "include": ["src/**/*.ts"] +} diff --git a/core-web/tsconfig.base.json b/core-web/tsconfig.base.json index 28336d565c77..bb8c83dfde96 100644 --- a/core-web/tsconfig.base.json +++ b/core-web/tsconfig.base.json @@ -69,6 +69,7 @@ "@dotcms/portlets/dot-locales/portlet/data-access": [ "libs/portlets/dot-locales/data-access/src/index.ts" ], + "@dotcms/portlets/dot-cdn/portlet": ["libs/portlets/dot-cdn/src/index.ts"], "@dotcms/portlets/dot-plugins/portlet": ["libs/portlets/dot-plugins/src/index.ts"], "@dotcms/portlets/dot-tags/portlet": ["libs/portlets/dot-tags/src/index.ts"], "@dotcms/portlets/dot-usage": ["libs/portlets/dot-usage/src/index.ts"], diff --git a/dotCMS/src/main/java/com/dotcms/cdn/CDNConstants.java b/dotCMS/src/main/java/com/dotcms/cdn/CDNConstants.java new file mode 100644 index 000000000000..b50015f09648 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/CDNConstants.java @@ -0,0 +1,13 @@ +package com.dotcms.cdn; + +public class CDNConstants { + + public static final String DOT_CDN_APP_KEY = "dotCDN"; + public static final String DOT_CDN_DOMAIN = "cdnDomain"; + public static final String DOT_CDN_ZONEID = "cdnZoneId"; + public static final String DOT_CDN_API_KEY = "cdnApiKey"; + + private CDNConstants() { + // constants class + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/CDNInterceptor.java b/dotCMS/src/main/java/com/dotcms/cdn/CDNInterceptor.java new file mode 100644 index 000000000000..38f6185479ec --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/CDNInterceptor.java @@ -0,0 +1,43 @@ +package com.dotcms.cdn; + +import java.util.Collections; +import java.util.List; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import com.dotcms.filters.interceptor.Result; +import com.dotcms.filters.interceptor.WebInterceptor; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; + +public class CDNInterceptor implements WebInterceptor { + + private static final long serialVersionUID = 1L; + private static final List SENSITIVE_HEADERS = List.of( + "authorization", "cookie", "proxy-authorization", "set-cookie", "x-csrf-token", + "x-xsrf-token"); + + @Override + public String[] getFilters() { + return new String[] {"/*"}; + } + + @Override + public Result intercept(final HttpServletRequest request, final HttpServletResponse response) { + + final boolean recordDotHeaders = request.getParameter("recordDotHeaders") != null + && Config.getBooleanProperty("DOT_CDN_DEBUG_HEADERS", false); + if (recordDotHeaders) { + final List headers = Collections.list(request.getHeaderNames()); + for (final String header : headers) { + Logger.info(this.getClass().getName(), + " CDN : " + header + " : " + headerValue(request, header)); + } + } + + return Result.NEXT; + } + + private String headerValue(final HttpServletRequest request, final String header) { + return SENSITIVE_HEADERS.contains(header.toLowerCase()) ? "[REDACTED]" : request.getHeader(header); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNAPI.java b/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNAPI.java new file mode 100644 index 000000000000..6ef691e4d692 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNAPI.java @@ -0,0 +1,85 @@ +package com.dotcms.cdn.api; + +import com.dotcms.cdn.CDNConstants; +import com.dotcms.security.apps.AppSecrets; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.util.UtilMethods; +import io.vavr.control.Try; + +import java.util.List; +import java.util.Optional; + +public interface DotCDNAPI { + + static DotCDNAPI api(Host host) { + return new DotCDNAPIImpl(host); + } + + static boolean isConfigured(final Host host) { + if (host == null || UtilMethods.isNotSet(host.getIdentifier())) { + return false; + } + + final Optional secrets = Try.of(() -> APILocator.getAppsAPI() + .getSecrets(CDNConstants.DOT_CDN_APP_KEY, true, host, APILocator.systemUser())) + .getOrElse(Optional.empty()); + return secrets.isPresent(); + } + + /** + * Logic to get and parse the Stats from bunny to {@link DotCDNStats} + * @param dateFromStr The start date of the statistics. + * @param dateToStr The end date of the statistics + * @return {@link DotCDNStats} + */ + default DotCDNStats getStats(final String dateFromStr, final String dateToStr) { + return getStats(dateFromStr, dateToStr, false); + } + + /** + * Logic to get and parse the Stats from bunny to {@link DotCDNStats} + * @param dateFromStr The start date of the statistics. + * @param dateToStr The end date of the statistics + * @param hourly If true, return hourly granularity instead of daily + * @return {@link DotCDNStats} + */ + DotCDNStats getStats(final String dateFromStr, final String dateToStr, final boolean hourly); + + /** + * Logic to invalidate the List of urls. + * @param urls List of url to invalidate + * @return true if all the urls were purged successfully, false if one or more failed. + */ + boolean invalidate(final List urls); + + /** + * Invalidate all urls related to the contentlet + * @param contentlet {@link Contentlet} + * @return boolean + */ + boolean invalidateContentlet(final Contentlet contentlet); + + /** + * Invalidate all urls related to the contentlet, in addition can pass extra urls to purge + * @param contentlet {@link Contentlet} + * @param urlsToPurge {@link List} + * @return boolean + */ + boolean invalidateContentlet(final Contentlet contentlet, final List urlsToPurge); + + /** + * Invalidate all pages urls related to the contentlet, in addition can pass extra urls to purge + * @param contentlet {@link Contentlet} + * @param urlsToPurge {@link List} + * @return boolean + */ + boolean invalidateRelatedPages(final Contentlet contentlet, final List urlsToPurge); + + /** + * Logic to invalidate the entire cache. + * @return true if the entire cache was invalidated successfully. + */ + boolean invalidateAll(); +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNAPIImpl.java b/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNAPIImpl.java new file mode 100644 index 000000000000..1d8ec5d8620b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNAPIImpl.java @@ -0,0 +1,408 @@ +package com.dotcms.cdn.api; + +import com.dotcms.cdn.CDNConstants; +import com.dotcms.http.CircuitBreakerUrl; +import com.dotcms.http.CircuitBreakerUrl.Method; +import com.dotcms.http.CircuitBreakerUrlBuilder; +import com.dotcms.rest.RestClientBuilder; +import com.dotcms.rest.exception.BadRequestException; +import com.dotcms.rest.exception.NotFoundException; +import com.dotcms.security.apps.AppSecrets; +import com.dotcms.uuid.shorty.ShortyIdAPI; +import com.dotmarketing.beans.Host; +import com.dotmarketing.beans.Identifier; +import com.dotmarketing.beans.MultiTree; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; +import com.dotmarketing.util.json.JSONObject; +import com.liferay.util.StringPool; +import io.vavr.control.Try; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +public class DotCDNAPIImpl implements DotCDNAPI { + + private final String accessKey; + private final long pullZoneId; + private final String cdnDomain; + + /** + * Method to load the app secrets into the variables. + * @param host if is not sent will throw IllegalArgumentException, if sent will try to + * find the secrets for it, if there is no secrets for the host will use the ones for the System_Host + */ + DotCDNAPIImpl(final Host host) { + if (host == null || UtilMethods.isNotSet(host.getIdentifier())) { + Logger.warn(DotCDNAPIImpl.class, "There is no host sent or found"); + throw new IllegalArgumentException("There is no host sent or found"); + } + final Optional appSecrets = Try.of(() -> APILocator.getAppsAPI() + .getSecrets(CDNConstants.DOT_CDN_APP_KEY, true, host, APILocator.systemUser())) + .getOrElse(Optional.empty()); + if (!appSecrets.isPresent()) { + Logger.warn(DotCDNAPIImpl.class, "There is no config set, please set it via Apps Tool"); + throw new NotFoundException("There is no config set, please set it via Apps Tool"); + } + this.accessKey = appSecrets.get().getSecrets().get(CDNConstants.DOT_CDN_API_KEY).getString(); + this.cdnDomain = appSecrets.get().getSecrets().get(CDNConstants.DOT_CDN_DOMAIN).getString(); + this.pullZoneId = Long.parseLong( + appSecrets.get().getSecrets().get(CDNConstants.DOT_CDN_ZONEID).getString()); + } + + /** + * Url from bunny api to get the stats + * @param from The start date of the statistics. + * @param to The end date of the statistics + * @return url from bunny api with the passed dates and the pullzone + */ + private static final DateTimeFormatter UTC_DATE_FORMATTER = + DateTimeFormatter.ISO_LOCAL_DATE.withZone(ZoneOffset.UTC); + + private String statsUrl(final Instant from, final Instant to, final boolean hourly) { + String url = "https://api.bunny.net/statistics?dateFrom=" + UTC_DATE_FORMATTER.format(from) + + "&dateTo=" + UTC_DATE_FORMATTER.format(to) + "&pullZone=" + pullZoneId + + "&loadErrors=true&loadOriginResponseTimes=true"; + if (hourly) { + url += "&hourly=true"; + } + Logger.debug(DotCDNAPIImpl.class, "Sending URL:" + url); + return url; + } + + CircuitBreakerUrlBuilder urlBuilder(final String url) { + return new CircuitBreakerUrlBuilder() + .setHeaders(Map.of("AccessKey", accessKey, "accept", "application/json")) + .setTimeout(10000) + .setUrl(url) + .setMethod(Method.GET); + } + + /** + * Method to get the response from the url that was hit. + * @return if the response was successful the object is returned, if not FAILURE is returned. + */ + private Map getData(CircuitBreakerUrlBuilder url) { + final String response = Try.of(() -> url.build().doString()) + .getOrElse("{\"response\":\"FAILURE\"}"); + final JSONObject jsonObject = new JSONObject(response); + return new HashMap<>(jsonObject); + } + + /** + * Logic to get and parse the Stats from bunny to {@link DotCDNStats} + * @param dateFromStr The start date of the statistics. + * @param dateToStr The end date of the statistics + * @return {@link DotCDNStats} + */ + @Override + public DotCDNStats getStats(final String dateFromStr, final String dateToStr, + final boolean hourly) { + final ZonedDateTime nowUtc = ZonedDateTime.now(ZoneOffset.UTC) + .withSecond(0).withMinute(0).withHour(0); + final Instant parsedFrom = Try.of( + () -> ZonedDateTime.parse(dateFromStr + "T00:00:00Z").toInstant()).getOrNull(); + final Instant parsedTo = Try.of( + () -> ZonedDateTime.parse(dateToStr + "T00:00:00Z").toInstant()).getOrNull(); + if (parsedFrom == null) { + Logger.debug(this.getClass().getName(), + "dateFrom is not sent or does not comply with the format yyyy-MM-dd, using date of 15 days ago"); + } + final Instant dateFrom = parsedFrom != null + ? parsedFrom : nowUtc.minusDays(15).toInstant(); + if (parsedTo == null) { + Logger.debug(this.getClass().getName(), + "dateTo is not sent or does not comply with the format yyyy-MM-dd, using today's date instead"); + } + final Instant dateTo = parsedTo != null ? parsedTo : nowUtc.toInstant(); + + final String statsUrl = statsUrl(dateFrom, dateTo, hourly); + final Map data = getData(urlBuilder(statsUrl)); + if ("FAILURE".equals(data.get("response"))) { + final String message = "Failed to get the Stats using the url: " + statsUrl + + " , please check the App Configuration"; + Logger.info(this.getClass().getName(), message); + throw new BadRequestException(message); + } + final DotCDNStats.DotStatsBuilder builder = DotCDNStats.builder() + .withCDNDomain(cdnDomain) + .withDateFrom(dateFrom.toString()) + .withDateTo(dateTo.toString()) + .withCacheHitRate(numberOrDefault(data.get("CacheHitRate"), 0).doubleValue()) + .withTotalRequestsServed(numberOrDefault(data.get("TotalRequestsServed"), 0).longValue()) + .withTotalBandwidthUsed(numberOrDefault(data.get("TotalBandwidthUsed"), 0).longValue()) + .withAverageOriginResponseTime( + numberOrDefault(data.get("AverageOriginResponseTime"), 0).intValue()); + + builder.withBandwidthUsedChart(mapOrEmpty(data, "BandwidthUsedChart")); + builder.withRequestsServedChart(mapOrEmpty(data, "RequestsServedChart")); + builder.withCacheHitRateChart(mapOrEmpty(data, "CacheHitRateChart")); + builder.withOriginResponseTimeChart(mapOrEmpty(data, "OriginResponseTimeChart")); + builder.withError4xxChart(mapOrEmpty(data, "Error4xxChart")); + builder.withError5xxChart(mapOrEmpty(data, "Error5xxChart")); + + @SuppressWarnings("unchecked") + final Map incomingGeo = + (Map) data.getOrDefault("GeoTrafficDistribution", + Collections.emptyMap()); + + final Map> geoMap = new LinkedHashMap<>(); + for (final String key : incomingGeo.keySet()) { + final String[] keySplit = key.split(":"); + if (keySplit.length < 2) { + continue; + } + final long traffic = numberOrDefault(incomingGeo.get(key), 0).longValue(); + final Map geoEntry = geoMap.computeIfAbsent(keySplit[0], + e -> new HashMap<>()); + geoEntry.put(keySplit[1], traffic); + } + builder.withGeographicDistribution(geoMap); + + return builder.build(); + } + + @Override + public boolean invalidateContentlet(final Contentlet contentlet) { + return this.invalidateContentlet(contentlet, Collections.emptyList()); + } + + @Override + public boolean invalidateContentlet(final Contentlet contentlet, + final List urlsToPurgeParam) { + + final List urlsToPurge = new ArrayList<>(); + final List contentletList = new ArrayList<>(); + contentletList.add(contentlet); + + if (UtilMethods.isSet(urlsToPurgeParam)) { + urlsToPurge.addAll(urlsToPurgeParam); + } + + if (contentlet.isHTMLPage()) { + contentletList.addAll(findPageContent(contentlet)); + } else { + contentletList.addAll(findPagesUsingContent(contentlet)); + } + + contentletList.forEach(contentletItem -> { + Try.of(() -> urlsToPurge.addAll(createUrlsToPurgeForContentlet(contentletItem))) + .onFailure(e -> Logger.warn(this.getClass().getName(), + "Unable to create urls to purge based on the contentlet: " + + e.getMessage())); + }); + + return this.invalidate(urlsToPurge); + } + + @Override + public boolean invalidateRelatedPages(final Contentlet contentlet, + final List urlsToPurgeParam) { + + final List urlsToPurge = new ArrayList<>(); + + if (UtilMethods.isSet(urlsToPurgeParam)) { + urlsToPurge.addAll(urlsToPurgeParam); + } + + findPagesUsingContent(contentlet).forEach(contentletItem -> { + Try.of(() -> urlsToPurge.addAll(createUrlsToPurgeForContentlet(contentletItem))) + .onFailure(e -> Logger.warn(this.getClass().getName(), + "Unable to create urls to purge based on the contentlet: " + + e.getMessage())); + }); + + return this.invalidate(urlsToPurge); + } + + /** + * Creates a List of strings (urls) based on the contentlet properties (path, identifier, inode). + * @param contentlet contentlet which the actionlet is being fired + * @return list of strings urls + */ + private List createUrlsToPurgeForContentlet(final Contentlet contentlet) { + + final List urlsForContentlet = new ArrayList<>(); + + final Identifier identifier = Try.of( + () -> APILocator.getIdentifierAPI().find(contentlet.getIdentifier())).getOrNull(); + if (identifier == null) { + Logger.debug(this.getClass().getName(), + "No identifier found for contentlet: " + contentlet.getIdentifier()); + return urlsForContentlet; + } + + final String path = identifier.getPath(); + urlsForContentlet.add(path); + + if (path.endsWith("/index")) { + urlsForContentlet.add(path.substring(0, path.length() - 5)); + urlsForContentlet.add(path.substring(0, path.length() - 6)); + } + + final ShortyIdAPI shorty = APILocator.getShortyAPI(); + final String inode = contentlet.getInode(); + final String id = identifier.getId(); + + urlsForContentlet.add(buildUrl(id, "*")); + urlsForContentlet.add(buildUrl(inode, "*")); + + urlsForContentlet.add(buildUrl(shorty.shortify(id), "*")); + urlsForContentlet.add(buildUrl(shorty.shortify(inode), "*")); + + final String urlMap = Try.of(() -> APILocator.getContentletAPI() + .getUrlMapForContentlet(contentlet, APILocator.systemUser(), false)).getOrNull(); + if (urlMap != null) { + urlsForContentlet.add(urlMap); + } + + urlsForContentlet.removeIf(UtilMethods::isEmpty); + + Logger.debug(this.getClass().getName(), + () -> "URLs to purge for contentlet: " + urlsForContentlet); + + return urlsForContentlet; + } + + private List findPagesUsingContent(final Contentlet contentlet) { + final List trees = Try.of( + () -> APILocator.getMultiTreeAPI().getMultiTreesByChild(contentlet.getIdentifier())) + .getOrElse(new ArrayList<>()); + + return trees.stream() + .map(tree -> Try.of(() -> APILocator.getContentletAPI() + .findContentletByIdentifierAnyLanguage(tree.getHtmlPage())).getOrNull()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private List findPageContent(final Contentlet contentlet) { + final List trees = Try.of( + () -> APILocator.getMultiTreeAPI().getMultiTreesByPage(contentlet.getIdentifier())) + .getOrElse(new ArrayList<>()); + + return trees.stream() + .map(tree -> Try.of(() -> APILocator.getContentletAPI() + .findContentletByIdentifierAnyLanguage(tree.getContentlet())).getOrNull()) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private String buildUrl(final String... concat) { + return "/dA/" + String.join(StringPool.SLASH, concat); + } + + /** + * Logic to invalidate the List of urls. + * @param urls List of url to invalidate + * @return true if all the urls were purged successfully, false if one or more failed. + */ + @Override + public boolean invalidate(final List urls) { + boolean results = true; + for (final String url : urls) { + + final String urlToPurge = url.startsWith(cdnDomain) ? url : + url.startsWith("/") ? cdnDomain + url : + cdnDomain + "/" + url; + + Logger.debug(this.getClass().getName(), "Purging URL: " + urlToPurge); + + final CircuitBreakerUrl.Response response = Try.of(() -> + this.urlBuilder(invalidateUrl(urlToPurge)) + .setMethod(Method.POST) + .setThrowWhenError(false) + .build() + .doResponse() + ).getOrNull(); + + if (response == null) { + Logger.warn(this.getClass().getName(), + "Purge failed (no response) for: " + urlToPurge); + results = false; + } else if (!CircuitBreakerUrl.isSuccessResponse(response.getStatusCode())) { + Logger.warn(this.getClass().getName(), + "Purge failed for: " + urlToPurge + + " - HTTP " + response.getStatusCode() + + " - " + response.getResponse()); + results = false; + } else { + Logger.debug(this.getClass().getName(), "Purge successful for: " + urlToPurge); + } + } + return results; + } + + /** + * Logic to invalidate the entire cache. + * @return true if the entire cache was invalidated successfully. + */ + @Override + public boolean invalidateAll() { + try (final Response response = RestClientBuilder.newClient().target(invalidateAllUrl()) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("AccessKey", accessKey) + .post(Entity.entity("", MediaType.TEXT_PLAIN))) { + Logger.debug(this.getClass().getName(), "Response: " + response); + if (response.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { + final String message = "Failed to Purge the entire Cache: " + invalidateAllUrl() + + " , please check the App Configuration"; + Logger.info(this.getClass().getName(), message); + throw new BadRequestException(message); + } + return true; + } + } + + private String invalidateAllUrl() { + return "https://api.bunny.net/pullzone/" + this.pullZoneId + "/purgeCache"; + } + + private String invalidateUrl(String url) { + try { + return "https://api.bunny.net/purge?url=" + + URLEncoder.encode(url, java.nio.charset.StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("UTF-8 encoding is not available", e); + } + } + + String getCdnDomain() { + return cdnDomain; + } + + private static Number numberOrDefault(final Object value, final Number defaultValue) { + if (value instanceof Number) { + return (Number) value; + } + return defaultValue; + } + + @SuppressWarnings("unchecked") + private static Map mapOrEmpty(final Map data, + final String key) { + final Object value = data.get(key); + if (value instanceof Map) { + return (Map) value; + } + return Collections.emptyMap(); + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNStats.java b/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNStats.java new file mode 100644 index 000000000000..c0180ec0491f --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/api/DotCDNStats.java @@ -0,0 +1,253 @@ +package com.dotcms.cdn.api; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.Collections; +import java.util.Map; +import javax.annotation.Nonnull; + +@JsonDeserialize(builder = DotCDNStats.DotStatsBuilder.class) +public class DotCDNStats { + + private final String cdnDomain; + private final String dateFrom; + private final String dateTo; + private final double cacheHitRate; + private final long totalBandwidthUsed; + private final long totalRequestsServed; + private final String bandwidthPretty; + private final int averageOriginResponseTime; + private final Map> geographicDistribution; + private final Map bandwidthUsedChart; + private final Map requestsServedChart; + private final Map cacheHitRateChart; + private final Map originResponseTimeChart; + private final Map error4xxChart; + private final Map error5xxChart; + + private DotCDNStats(DotStatsBuilder builder) { + this.cdnDomain = builder.cdnDomain; + this.dateFrom = builder.dateFrom.toString(); + this.dateTo = builder.dateTo.toString(); + this.cacheHitRate = builder.cacheHitRate; + this.totalBandwidthUsed = builder.totalBandwidthUsed; + this.totalRequestsServed = builder.totalRequestsServed; + this.averageOriginResponseTime = builder.averageOriginResponseTime; + this.geographicDistribution = builder.geographicDistribution; + this.bandwidthPretty = prettyByteify(builder.totalBandwidthUsed); + this.bandwidthUsedChart = builder.bandwidthUsedChart; + this.requestsServedChart = builder.requestsServedChart; + this.cacheHitRateChart = builder.cacheHitRateChart; + this.originResponseTimeChart = builder.originResponseTimeChart; + this.error4xxChart = builder.error4xxChart; + this.error5xxChart = builder.error5xxChart; + } + + public static DotStatsBuilder builder() { + return new DotStatsBuilder(); + } + + public static DotStatsBuilder from(final DotCDNStats dotCDNStats) { + return new DotStatsBuilder(dotCDNStats); + } + + public String getCdnDomain() { + return cdnDomain; + } + + public String getDateFrom() { + return dateFrom; + } + + public String getDateTo() { + return dateTo; + } + + public double getCacheHitRate() { + return cacheHitRate; + } + + public long getTotalBandwidthUsed() { + return totalBandwidthUsed; + } + + public long getTotalRequestsServed() { + return totalRequestsServed; + } + + public String getBandwidthPretty() { + return bandwidthPretty; + } + + public int getAverageOriginResponseTime() { + return averageOriginResponseTime; + } + + public Map> getGeographicDistribution() { + return geographicDistribution; + } + + public Map getBandwidthUsedChart() { + return bandwidthUsedChart; + } + + public Map getRequestsServedChart() { + return requestsServedChart; + } + + public Map getCacheHitRateChart() { + return cacheHitRateChart; + } + + public Map getOriginResponseTimeChart() { + return originResponseTimeChart; + } + + public Map getError4xxChart() { + return error4xxChart; + } + + public Map getError5xxChart() { + return error5xxChart; + } + + public static final class DotStatsBuilder { + + private String cdnDomain; + private String dateFrom; + private String dateTo; + private double cacheHitRate; + private long totalBandwidthUsed; + private long totalRequestsServed; + private int averageOriginResponseTime; + private Map> geographicDistribution = Collections.emptyMap(); + private Map bandwidthUsedChart; + private Map requestsServedChart; + private Map cacheHitRateChart = Collections.emptyMap(); + private Map originResponseTimeChart = Collections.emptyMap(); + private Map error4xxChart = Collections.emptyMap(); + private Map error5xxChart = Collections.emptyMap(); + + private DotStatsBuilder() {} + + private DotStatsBuilder(final DotCDNStats dotCDNStats) { + this.cdnDomain = dotCDNStats.cdnDomain; + this.dateFrom = dotCDNStats.dateFrom; + this.dateTo = dotCDNStats.dateTo; + this.cacheHitRate = dotCDNStats.cacheHitRate; + this.totalBandwidthUsed = dotCDNStats.totalBandwidthUsed; + this.totalRequestsServed = dotCDNStats.totalRequestsServed; + this.averageOriginResponseTime = dotCDNStats.averageOriginResponseTime; + this.geographicDistribution = dotCDNStats.geographicDistribution; + this.bandwidthUsedChart = dotCDNStats.bandwidthUsedChart; + this.requestsServedChart = dotCDNStats.requestsServedChart; + this.cacheHitRateChart = dotCDNStats.cacheHitRateChart; + this.originResponseTimeChart = dotCDNStats.originResponseTimeChart; + this.error4xxChart = dotCDNStats.error4xxChart; + this.error5xxChart = dotCDNStats.error5xxChart; + } + + public DotStatsBuilder withCDNDomain(@Nonnull final String cdnDomain) { + this.cdnDomain = cdnDomain; + return this; + } + + public DotStatsBuilder withDateFrom(@Nonnull final String dateFrom) { + this.dateFrom = dateFrom; + return this; + } + + public DotStatsBuilder withDateTo(@Nonnull final String dateTo) { + this.dateTo = dateTo; + return this; + } + + public DotStatsBuilder withCacheHitRate(final double cacheHitRate) { + this.cacheHitRate = cacheHitRate; + return this; + } + + public DotStatsBuilder withTotalBandwidthUsed(final long totalBandwidthUsed) { + this.totalBandwidthUsed = totalBandwidthUsed; + return this; + } + + public DotStatsBuilder withTotalRequestsServed(final long totalRequestsServed) { + this.totalRequestsServed = totalRequestsServed; + return this; + } + + public DotStatsBuilder withAverageOriginResponseTime(final int averageOriginResponseTime) { + this.averageOriginResponseTime = averageOriginResponseTime; + return this; + } + + public DotStatsBuilder withGeographicDistribution( + @Nonnull final Map> geographicDistribution) { + this.geographicDistribution = geographicDistribution; + return this; + } + + public DotStatsBuilder withBandwidthUsedChart( + @Nonnull final Map bandWidthUsed) { + this.bandwidthUsedChart = bandWidthUsed; + return this; + } + + public DotStatsBuilder withRequestsServedChart( + @Nonnull final Map requestsServedChart) { + this.requestsServedChart = requestsServedChart; + return this; + } + + public DotStatsBuilder withCacheHitRateChart( + @Nonnull final Map cacheHitRateChart) { + this.cacheHitRateChart = cacheHitRateChart; + return this; + } + + public DotStatsBuilder withOriginResponseTimeChart( + @Nonnull final Map originResponseTimeChart) { + this.originResponseTimeChart = originResponseTimeChart; + return this; + } + + public DotStatsBuilder withError4xxChart( + @Nonnull final Map error4xxChart) { + this.error4xxChart = error4xxChart; + return this; + } + + public DotStatsBuilder withError5xxChart( + @Nonnull final Map error5xxChart) { + this.error5xxChart = error5xxChart; + return this; + } + + public DotCDNStats build() { + return new DotCDNStats(this); + } + } + + private static final int BYTES_PER_DECIMAL_UNIT = 1000; + + private String prettyByteify(final long memory) { + final NumberFormat numberFormat = new DecimalFormat("#.00"); + final double x = memory; + if (x > (BYTES_PER_DECIMAL_UNIT * BYTES_PER_DECIMAL_UNIT * BYTES_PER_DECIMAL_UNIT)) { + return numberFormat.format(x / (BYTES_PER_DECIMAL_UNIT * BYTES_PER_DECIMAL_UNIT + * BYTES_PER_DECIMAL_UNIT)) + " GB"; + } else if (x > (BYTES_PER_DECIMAL_UNIT * BYTES_PER_DECIMAL_UNIT)) { + return numberFormat.format(x / (BYTES_PER_DECIMAL_UNIT * BYTES_PER_DECIMAL_UNIT)) + + " MB"; + } else if (x > BYTES_PER_DECIMAL_UNIT) { + return numberFormat.format(x / BYTES_PER_DECIMAL_UNIT) + " KB"; + } else if (x > 1) { + return numberFormat.format(x) + " B"; + } else { + return "0 b"; + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/pushpublish/receiver/event/PushPublishOnReceiverEndSubscriber.java b/dotCMS/src/main/java/com/dotcms/cdn/pushpublish/receiver/event/PushPublishOnReceiverEndSubscriber.java new file mode 100644 index 000000000000..05cdd565fdd8 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/pushpublish/receiver/event/PushPublishOnReceiverEndSubscriber.java @@ -0,0 +1,140 @@ +package com.dotcms.cdn.pushpublish.receiver.event; + +import com.dotcms.cdn.api.DotCDNAPI; +import com.dotcms.publisher.business.PublishQueueElement; +import com.dotcms.system.event.local.model.Subscriber; +import com.dotcms.system.event.local.type.pushpublish.receiver.PushPublishEndOnReceiverEvent; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.portlets.contentlet.business.ContentletAPI; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import com.liferay.portal.model.User; +import io.vavr.control.Try; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * This subscriber is in charge of invalidating CDN cache when push-publish completes on receiver. + */ +public class PushPublishOnReceiverEndSubscriber { + + private static final boolean LIVE = false; + private static final boolean RESPECT_FRONTEND_ROLES = false; + + private final User user = APILocator.systemUser(); + + @Subscriber + public void notify(PushPublishEndOnReceiverEvent event) { + + final ContentletAPI contentletAPI = APILocator.getContentletAPI(); + final List publishQueueElements = + event.getPublishQueueElements().stream() + .filter(pqe -> "contentlet".equals(pqe.getType())) + .collect(Collectors.toList()); + final int minimumNumberToPurgeAll = + Config.getIntProperty("DOT_CDN_MIN_NUM_TO_PURGE_ALL", 250); + + if (publishQueueElements.size() > minimumNumberToPurgeAll) { + this.invalidateAllSites(contentletAPI, publishQueueElements); + } else { + invalidateContentlets(contentletAPI, publishQueueElements); + } + } + + private void invalidateContentlets(final ContentletAPI contentletAPI, + final List publishQueueElements) { + + final Map dotCDNAPIMap = new HashMap<>(); + final Set unconfiguredHosts = new HashSet<>(); + Logger.info(this, "Purging selective Contentlets on CDN"); + + for (final PublishQueueElement publishQueueElement : publishQueueElements) { + + final String identifier = publishQueueElement.getAsset(); + final long languageId = publishQueueElement.getLanguageId(); + final Contentlet contentlet = Try.of(() -> contentletAPI + .findContentletByIdentifier(identifier, LIVE, languageId, user, + RESPECT_FRONTEND_ROLES)).getOrNull(); + + if (null == contentlet) { + Logger.debug(this, () -> "Cannot invalidate contentlet: " + + identifier + " — not found"); + continue; + } + + final String hostId = contentlet.getHost(); + if (unconfiguredHosts.contains(hostId)) { + continue; + } + + final Host site = Try.of( + () -> APILocator.getHostAPI().find(hostId, user, RESPECT_FRONTEND_ROLES)) + .getOrNull(); + if (null == site) { + Logger.debug(this, () -> "Cannot invalidate contentlet: " + + identifier + " — host not found: " + hostId); + continue; + } + + if (!DotCDNAPI.isConfigured(site)) { + unconfiguredHosts.add(hostId); + Logger.debug(this, () -> "dotCDN not configured for host: " + + site.getHostname() + ", skipping"); + continue; + } + + final DotCDNAPI cdnApi = + dotCDNAPIMap.computeIfAbsent(hostId, k -> DotCDNAPI.api(site)); + Logger.debug(this, () -> "Invalidating contentlet: " + + contentlet.getIdentifier()); + Try.run(() -> cdnApi.invalidateContentlet(contentlet)) + .onFailure(e -> Logger.warn(this, + "CDN purge failed for contentlet: " + contentlet.getIdentifier() + + " - " + e.getMessage())); + } + } + + private void invalidateAllSites(final ContentletAPI contentletAPI, + final List publishQueueElements) { + + final Map hostMap = new HashMap<>(); + for (final PublishQueueElement publishQueueElement : publishQueueElements) { + + final String identifier = publishQueueElement.getAsset(); + final long languageId = publishQueueElement.getLanguageId(); + final Contentlet contentlet = Try.of(() -> contentletAPI + .findContentletByIdentifier(identifier, LIVE, languageId, user, + RESPECT_FRONTEND_ROLES)).getOrNull(); + if (null != contentlet) { + final Host site = Try.of(() -> APILocator.getHostAPI() + .find(contentlet.getHost(), user, RESPECT_FRONTEND_ROLES)).getOrNull(); + if (null != site) { + hostMap.put(contentlet.getHost(), site); + } + } + } + + for (final Host site : hostMap.values()) { + if (!DotCDNAPI.isConfigured(site)) { + Logger.debug(this, () -> "dotCDN not configured for host: " + + site.getHostname() + ", skipping purge-all"); + continue; + } + + final DotCDNAPI cdnApi = DotCDNAPI.api(site); + Logger.info(this, "Purging all CDN, host: " + site.getHostname()); + Try.of(cdnApi::invalidateAll) + .onSuccess(resultInvalidate -> Logger.info(this, "Purge result: " + + resultInvalidate + " host: " + site.getHostname())) + .onFailure(e -> Logger.warn(this, "Unable to purge all CDN, host: " + + site.getHostname() + " - " + e.getMessage())); + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/rest/DotCDNResource.java b/dotCMS/src/main/java/com/dotcms/cdn/rest/DotCDNResource.java new file mode 100644 index 000000000000..0fb5c13c3caa --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/rest/DotCDNResource.java @@ -0,0 +1,249 @@ +package com.dotcms.cdn.rest; + +import com.dotcms.cdn.api.DotCDNAPI; +import com.dotcms.cdn.api.DotCDNStats; +import com.dotcms.rest.ResponseEntityView; +import com.dotcms.rest.WebResource; +import com.dotcms.rest.annotation.NoCache; +import com.dotcms.rest.exception.BadRequestException; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; +import com.dotmarketing.util.WebKeys; +import com.liferay.portal.model.User; +import io.vavr.Lazy; +import io.vavr.control.Try; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; + +import java.util.List; +import java.util.stream.Collectors; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Path("/v1/dotcdn") +public class DotCDNResource { + + private static final int DEFAULT_MAX_PURGE_URLS = 1000; + private static final int MAX_PURGE_URL_LENGTH = 2048; + + /** + * This endpoint is to get the statistics from the CDN. + * We get the stats of a range of dates. + * + * The hostId property is to get the AppSecret Configuration. If the hostId is not send will + * try to get it from the session. + * + * @param dateFromStr date from we should get the stats, format yyyy-MM-dd + * @param dateToStr date until we should get the stats, format: yyyy-MM-dd + * @param hostId Id of the host which App config we should get. + * @return stats response + */ + @GET + @Path("/stats") + @NoCache + @Produces(MediaType.APPLICATION_JSON) + @Operation( + summary = "Get CDN statistics", + responses = @ApiResponse( + responseCode = "200", + description = "CDN statistics for the requested date range", + content = @Content( + schema = @Schema(implementation = ResponseEntityDotCDNStatsView.class)))) + public final Response getStats( + @Context final HttpServletRequest request, + @Context final HttpServletResponse response, + @QueryParam("dateFrom") final String dateFromStr, + @QueryParam("dateTo") final String dateToStr, + @QueryParam("hostId") final String hostId, + @QueryParam("hourly") final boolean hourly) { + + final User user = new WebResource.InitBuilder(request, response) + .rejectWhenNoUser(true).requiredBackendUser(true) + .requiredPortlet("dotCDN").init().getUser(); + + final Lazy lazyCurrentHost = Lazy.of(() -> Try.of( + () -> Host.class.cast(request.getSession().getAttribute(WebKeys.CURRENT_HOST))) + .getOrNull()); + final Host host = Try.of(() -> APILocator.getHostAPI() + .find(hostId, user, false)).getOrElse(lazyCurrentHost.get()); + + if (!DotCDNAPI.isConfigured(host)) { + throw new BadRequestException( + "dotCDN is not configured for this host. Please configure it via the Apps tool."); + } + + return Response.ok(new ResponseEntityDotCDNStatsView( + new DotCDNStatsResponse(DotCDNAPI.api(host).getStats(dateFromStr, dateToStr, hourly)))) + .build(); + } + + /** + * This endpoint is to purgeCache of the CDN. + * You can purge the entire cache by setting the invalidateAll to true + * Or you can purge specific urls by sending them in a Array. + * + * The hostId property is to get the AppSecret Configuration. If the hostId is not send will + * try to get it from the session. + * + * @param invalidationForm request body + * @return a message if the purged was successful. + */ + @DELETE + @Path("/") + @NoCache + @Produces(MediaType.APPLICATION_JSON) + @Operation( + summary = "Purge CDN cache", + responses = @ApiResponse( + responseCode = "200", + description = "CDN purge result", + content = @Content( + schema = @Schema(implementation = ResponseEntityDotCDNPurgeView.class)))) + public final Response purgeCache(@Context final HttpServletRequest request, + @Context final HttpServletResponse response, + final InvalidationForm invalidationForm) { + + final User user = new WebResource.InitBuilder(request, response) + .rejectWhenNoUser(true).requiredBackendUser(true) + .requiredPortlet("dotCDN").init().getUser(); + + final Lazy lazyCurrentHost = Lazy.of(() -> Try.of( + () -> Host.class.cast(request.getSession().getAttribute(WebKeys.CURRENT_HOST))) + .getOrNull()); + if (invalidationForm == null) { + throw new BadRequestException("Invalidation form is required"); + } + + final Host host = Try.of(() -> APILocator.getHostAPI() + .find(invalidationForm.getHostId(), user, false)) + .getOrElse(lazyCurrentHost.get()); + + if (!DotCDNAPI.isConfigured(host)) { + throw new BadRequestException( + "dotCDN is not configured for this host. Please configure it via the Apps tool."); + } + + final DotCDNAPI cdnApi = DotCDNAPI.api(host); + + if (invalidationForm.isInvalidateAll()) { + Logger.info(this.getClass().getName(), + "User:" + user.getUserId() + " purging entire cache"); + return Response.ok(new ResponseEntityDotCDNPurgeView( + new DotCDNPurgeResponse(cdnApi.invalidateAll(), true))).build(); + } + + final List urls = validateUrls(invalidationForm.getUrls()); + return Response.ok(new ResponseEntityDotCDNPurgeView( + new DotCDNPurgeResponse(cdnApi.invalidate(urls), false))).build(); + } + + private List validateUrls(final List urls) { + if (!UtilMethods.isSet(urls)) { + throw new BadRequestException("At least one URL is required"); + } + + final int maxPurgeUrls = Config.getIntProperty( + "DOT_CDN_MAX_PURGE_URLS", DEFAULT_MAX_PURGE_URLS); + if (urls.size() > maxPurgeUrls) { + throw new BadRequestException("A maximum of " + maxPurgeUrls + " URLs can be purged"); + } + + final List normalizedUrls = urls.stream() + .filter(UtilMethods::isSet) + .map(String::trim) + .collect(Collectors.toList()); + + if (normalizedUrls.size() != urls.size()) { + throw new BadRequestException("Purge URLs cannot be blank"); + } + + for (final String url : normalizedUrls) { + if (!isValidPurgeUrl(url)) { + throw new BadRequestException("Invalid purge URL: " + url); + } + } + + return normalizedUrls; + } + + private boolean isValidPurgeUrl(final String url) { + if (url.length() > MAX_PURGE_URL_LENGTH || containsControlCharacter(url)) { + return false; + } + + return url.startsWith("/"); + } + + private boolean containsControlCharacter(final String url) { + for (int i = 0; i < url.length(); i++) { + if (Character.isISOControl(url.charAt(i))) { + return true; + } + } + + return false; + } + + @Schema(description = "CDN statistics response") + public static class DotCDNStatsResponse { + + private final DotCDNStats stats; + + public DotCDNStatsResponse(final DotCDNStats stats) { + this.stats = stats; + } + + public DotCDNStats getStats() { + return stats; + } + } + + @Schema(description = "CDN purge response") + public static class DotCDNPurgeResponse { + + private final boolean success; + private final boolean invalidateAll; + + public DotCDNPurgeResponse(final boolean success, final boolean invalidateAll) { + this.success = success; + this.invalidateAll = invalidateAll; + } + + public boolean isSuccess() { + return success; + } + + public boolean isInvalidateAll() { + return invalidateAll; + } + } + + public static class ResponseEntityDotCDNStatsView + extends ResponseEntityView { + + public ResponseEntityDotCDNStatsView(final DotCDNStatsResponse entity) { + super(entity); + } + } + + public static class ResponseEntityDotCDNPurgeView + extends ResponseEntityView { + + public ResponseEntityDotCDNPurgeView(final DotCDNPurgeResponse entity) { + super(entity); + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/rest/InvalidationForm.java b/dotCMS/src/main/java/com/dotcms/cdn/rest/InvalidationForm.java new file mode 100644 index 000000000000..fc13a9d1b5ad --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/rest/InvalidationForm.java @@ -0,0 +1,64 @@ +package com.dotcms.cdn.rest; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@JsonDeserialize(builder = InvalidationForm.Builder.class) +public class InvalidationForm { + + private final List urls; + private final boolean invalidateAll; + private final String hostId; + + private InvalidationForm(Builder builder) { + urls = Collections.unmodifiableList(new ArrayList<>(builder.urls)); + invalidateAll = builder.invalidateAll; + hostId = builder.hostId; + } + + public boolean isInvalidateAll() { + return invalidateAll; + } + + public String getHostId() { + return hostId; + } + + public List getUrls() { + return urls; + } + + public static final class Builder { + + @JsonProperty + private List urls = new ArrayList<>(); + + @JsonProperty + private String hostId; + + @JsonProperty + private boolean invalidateAll = false; + + public Builder urls(final List urls) { + this.urls = urls == null ? new ArrayList<>() : new ArrayList<>(urls); + return this; + } + + public Builder invalidateAll(final boolean invalidateAll) { + this.invalidateAll = invalidateAll; + return this; + } + + public Builder hostId(final String hostId) { + this.hostId = hostId; + return this; + } + + public InvalidationForm build() { + return new InvalidationForm(this); + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/viewtool/DotCDNTool.java b/dotCMS/src/main/java/com/dotcms/cdn/viewtool/DotCDNTool.java new file mode 100644 index 000000000000..36953029d0fb --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/viewtool/DotCDNTool.java @@ -0,0 +1,45 @@ +package com.dotcms.cdn.viewtool; + +import java.util.Optional; +import javax.servlet.http.HttpServletRequest; +import org.apache.velocity.tools.view.context.ViewContext; +import org.apache.velocity.tools.view.tools.ViewTool; +import com.dotcms.cdn.CDNConstants; +import com.dotcms.security.apps.AppSecrets; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.business.web.WebAPILocator; +import io.vavr.Lazy; +import io.vavr.control.Try; + +public class DotCDNTool implements ViewTool { + + private HttpServletRequest request; + private Host host; + + private final Lazy> appSecrets = Lazy.of(() -> + Try.of(() -> APILocator.getAppsAPI() + .getSecrets(CDNConstants.DOT_CDN_APP_KEY, true, host, APILocator.systemUser())) + .getOrElse(Optional.empty())); + + @Override + public void init(Object initData) { + this.request = ((ViewContext) initData).getRequest(); + this.host = WebAPILocator.getHostWebAPI().getCurrentHostNoThrow(this.request); + } + + /** + * Creates the full url where the file lives in the cdn. + * + * @return the cdn domain + url, that is where the file lives + */ + public String cdnify(final String fileUrl) { + if (!appSecrets.get().isPresent() || fileUrl == null + || fileUrl.contains("//") || !fileUrl.startsWith("/")) { + return fileUrl; + } + + return appSecrets.get().get().getSecrets() + .get(CDNConstants.DOT_CDN_DOMAIN).getString() + fileUrl; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/viewtool/DotCDNToolInfo.java b/dotCMS/src/main/java/com/dotcms/cdn/viewtool/DotCDNToolInfo.java new file mode 100644 index 000000000000..7cbb72ccb4e2 --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/viewtool/DotCDNToolInfo.java @@ -0,0 +1,30 @@ +package com.dotcms.cdn.viewtool; + +import org.apache.velocity.tools.view.context.ViewContext; +import org.apache.velocity.tools.view.servlet.ServletToolInfo; + +public class DotCDNToolInfo extends ServletToolInfo { + + @Override + public String getKey() { + return "dotcdn"; + } + + @Override + public String getScope() { + return ViewContext.REQUEST; + } + + @Override + public String getClassname() { + return DotCDNTool.class.getName(); + } + + @Override + public Object getInstance(Object initData) { + DotCDNTool viewTool = new DotCDNTool(); + viewTool.init(initData); + setScope(ViewContext.REQUEST); + return viewTool; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/cdn/workflow/DotCDNInvalidateActionlet.java b/dotCMS/src/main/java/com/dotcms/cdn/workflow/DotCDNInvalidateActionlet.java new file mode 100644 index 000000000000..cd17a39290ce --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/cdn/workflow/DotCDNInvalidateActionlet.java @@ -0,0 +1,112 @@ +package com.dotcms.cdn.workflow; + +import com.dotcms.cdn.api.DotCDNAPI; +import com.dotcms.concurrent.DotConcurrentFactory; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.portlets.workflows.actionlet.WorkFlowActionlet; +import com.dotmarketing.portlets.workflows.model.WorkflowActionClassParameter; +import com.dotmarketing.portlets.workflows.model.WorkflowActionFailureException; +import com.dotmarketing.portlets.workflows.model.WorkflowActionletParameter; +import com.dotmarketing.portlets.workflows.model.WorkflowProcessor; +import com.dotmarketing.util.Logger; +import com.dotmarketing.util.UtilMethods; +import io.vavr.control.Try; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.StringTokenizer; + +public class DotCDNInvalidateActionlet extends WorkFlowActionlet { + + private static final long serialVersionUID = 1L; + private static final String PURGE_CONTENTLET_PARAM = "purgeContentlet"; + private static final String PURGE_URL_PARAM = "purgeUrl"; + + @Override + public List getParameters() { + final List params = new ArrayList<>(); + params.add(new WorkflowActionletParameter(PURGE_CONTENTLET_PARAM, + "Purge Contentlet", "true", false)); + params.add(new WorkflowActionletParameter(PURGE_URL_PARAM, + "Additional Url(s) to Purge", "", false)); + return params; + } + + @Override + public String getName() { + return "dotCDN Purge"; + } + + @Override + public String getHowTo() { + return "URL(s) to purge: can be set to specific url(s) to be purged. " + + "Use comma (,) as a delimiter.
" + + "Purge Contentlet: It creates a list of patterns that should " + + "be purged using the $contentlet object." + + "e.g. /dA/$contentlet.identifier/* or /dA/$contentlet.shortyInode/*"; + } + + @Override + public void executeAction(WorkflowProcessor processor, + Map params) + throws WorkflowActionFailureException { + + final Contentlet contentlet = processor.getContentlet(); + final String purgeUrls = params.get(PURGE_URL_PARAM).getValue(); + final boolean isPurgeContentlet = Try.of( + () -> Boolean.parseBoolean( + params.get(PURGE_CONTENTLET_PARAM).getValue().trim())) + .getOrElse(true); + final Host host = Try.of(() -> APILocator.getHostAPI() + .find(contentlet.getHost(), APILocator.systemUser(), false)) + .getOrNull(); + if (host == null) { + Logger.warn(this.getClass().getName(), "Contentlet Host is Null"); + return; + } + if (!DotCDNAPI.isConfigured(host)) { + Logger.debug(this.getClass().getName(), + "dotCDN not configured for host: " + host.getHostname() + ", skipping"); + return; + } + final DotCDNAPI cdnApi = DotCDNAPI.api(host); + final List urlsToPurge = new ArrayList<>(); + + if (!UtilMethods.isEmpty(purgeUrls)) { + urlsToPurge.addAll(parseUrlsParam(purgeUrls)); + } + + Logger.info(this, "PurgeContentlet: " + isPurgeContentlet); + if (isPurgeContentlet) { + DotConcurrentFactory.getInstance().getSubmitter().submit(() -> { + Try.of(() -> cdnApi.invalidateContentlet(contentlet, urlsToPurge)) + .onFailure(e -> Logger.warn(DotCDNInvalidateActionlet.class, + "CDN purge failed for contentlet: " + + contentlet.getIdentifier() + " - " + e.getMessage())); + }); + } else { + DotConcurrentFactory.getInstance().getSubmitter().submit(() -> { + Try.of(() -> cdnApi.invalidateRelatedPages(contentlet, urlsToPurge)) + .onFailure(e -> Logger.warn(DotCDNInvalidateActionlet.class, + "CDN purge (related pages) failed for contentlet: " + + contentlet.getIdentifier() + " - " + e.getMessage())); + }); + } + } + + private List parseUrlsParam(final String urlsParam) { + final List urls = new ArrayList<>(); + if (urlsParam == null) { + return urls; + } + final StringTokenizer st = new StringTokenizer(urlsParam, ","); + while (st.hasMoreTokens()) { + urls.add(st.nextToken().trim()); + } + urls.removeIf(UtilMethods::isEmpty); + return urls; + } +} diff --git a/dotCMS/src/main/java/com/dotcms/rest/config/DotRestApplication.java b/dotCMS/src/main/java/com/dotcms/rest/config/DotRestApplication.java index f44ca0470f7f..c8e020315264 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/config/DotRestApplication.java +++ b/dotCMS/src/main/java/com/dotcms/rest/config/DotRestApplication.java @@ -16,8 +16,10 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.ws.rs.ApplicationPath; +import javax.ws.rs.Path; import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.server.ServerProperties; @@ -130,6 +132,14 @@ public class DotRestApplication extends ResourceConfig { private static final Lazy ENABLE_TELEMETRY_FROM_CORE = Lazy.of(() -> Config.getBooleanProperty(FeatureFlagName.FEATURE_FLAG_TELEMETRY_CORE_ENABLED, true)); + /** + * @Path values of REST resources registered via core package scanning. + * Used to reject OSGI plugins that would conflict with migrated-to-core resources. + */ + private static final Set CORE_REST_PATHS = Set.of( + "/v1/dotcdn" + ); + public DotRestApplication() { // Include the rest of the application configuration @@ -143,6 +153,7 @@ private void configureApplication() { "com.dotcms.contenttype.model.field", "com.dotcms.rendering.js", "com.dotcms.ai.rest", + "com.dotcms.cdn.rest", "com.dotcms.health", "io.swagger.v3.jaxrs2")); @@ -189,6 +200,18 @@ public static synchronized void addClass(final Class clazz) { return; } + // Check if a core resource already serves the same @Path — prevents OSGI plugins + // from conflicting with resources that have been migrated into core + final Path pluginPath = clazz.getAnnotation(Path.class); + if (pluginPath != null && CORE_REST_PATHS.contains(pluginPath.value())) { + Logger.warn(DotRestApplication.class, + "Plugin REST resource at @Path(\"" + pluginPath.value() + + "\") conflicts with a core resource, skipping: " + clazz.getName() + + ". If this was an OSGI plugin, it has been migrated into core" + + " and the plugin should be uninstalled."); + return; + } + // Add the new class and reload customClasses.put(clazz, true); Logger.info(DotRestApplication.class, diff --git a/dotCMS/src/main/java/com/dotcms/system/event/local/business/LocalSystemEventSubscribersInitializer.java b/dotCMS/src/main/java/com/dotcms/system/event/local/business/LocalSystemEventSubscribersInitializer.java index 1e27f3f8436a..31d3b6a75ebf 100644 --- a/dotCMS/src/main/java/com/dotcms/system/event/local/business/LocalSystemEventSubscribersInitializer.java +++ b/dotCMS/src/main/java/com/dotcms/system/event/local/business/LocalSystemEventSubscribersInitializer.java @@ -1,5 +1,6 @@ package com.dotcms.system.event.local.business; +import com.dotcms.cdn.pushpublish.receiver.event.PushPublishOnReceiverEndSubscriber; import com.dotcms.ai.listener.AIAppListener; import com.dotcms.analytics.listener.AnalyticsAppListener; import com.dotcms.config.DotInitializer; @@ -72,6 +73,8 @@ public void notify(final ChangeLoggerLevelEvent event) { APILocator.getLocalSystemEventsAPI().subscribe(AppSecretSavedEvent.class, AnalyticsAppListener.Instance.get()); APILocator.getLocalSystemEventsAPI().subscribe(AppSecretSavedEvent.class, AIAppListener.Instance.get()); + APILocator.getLocalSystemEventsAPI().subscribe(new PushPublishOnReceiverEndSubscriber()); + this.initDotVelocityMacrosVtlFiles(); } diff --git a/dotCMS/src/main/java/com/dotmarketing/filters/InterceptorFilter.java b/dotCMS/src/main/java/com/dotmarketing/filters/InterceptorFilter.java index 700a3cdbf4ba..e88b41731e8c 100644 --- a/dotCMS/src/main/java/com/dotmarketing/filters/InterceptorFilter.java +++ b/dotCMS/src/main/java/com/dotmarketing/filters/InterceptorFilter.java @@ -1,6 +1,7 @@ package com.dotmarketing.filters; import com.dotcms.analytics.track.AnalyticsTrackWebInterceptor; +import com.dotcms.cdn.CDNInterceptor; import com.dotcms.business.SystemTableUpdatedKeyEvent; import com.dotcms.ema.EMAWebInterceptor; import com.dotcms.filters.interceptor.AbstractWebInterceptorSupportFilter; @@ -56,6 +57,7 @@ private void addInterceptors(final FilterConfig config) { delegate.add(new EventLogWebInterceptor()); delegate.add(new CurrentVariantWebInterceptor()); delegate.add(analyticsTrackWebInterceptor); + delegate.add(new CDNInterceptor()); APILocator.getLocalSystemEventsAPI().subscribe(SystemTableUpdatedKeyEvent.class, analyticsTrackWebInterceptor); } // addInterceptors. diff --git a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowAPIImpl.java b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowAPIImpl.java index bbd8876e840b..1797d359f42a 100644 --- a/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowAPIImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/portlets/workflows/business/WorkflowAPIImpl.java @@ -22,6 +22,7 @@ import com.dotcms.exception.ExceptionUtil; import com.dotcms.notifications.bean.NotificationLevel; import com.dotcms.notifications.bean.NotificationType; +import com.dotcms.cdn.workflow.DotCDNInvalidateActionlet; import com.dotcms.rekognition.actionlet.RekognitionActionlet; import com.dotcms.rendering.js.JsScriptActionlet; import com.google.common.annotations.VisibleForTesting; @@ -295,7 +296,8 @@ public WorkflowAPIImpl() { OpenAIAutoTagActionlet.class, AnalyticsFireUserEventActionlet.class, OpenAIVisionAutoTagActionlet.class, - OpenAITranslationActionlet.class + OpenAITranslationActionlet.class, + DotCDNInvalidateActionlet.class )); refreshWorkFlowActionletMap(); diff --git a/dotCMS/src/main/resources/apps/dotCDN.yml b/dotCMS/src/main/resources/apps/dotCDN.yml new file mode 100644 index 000000000000..3e09a1b257b8 --- /dev/null +++ b/dotCMS/src/main/resources/apps/dotCDN.yml @@ -0,0 +1,30 @@ +--- +name: "dotCDN" +description: "This app connects your dotCMS instance with dotCMS's CDN for faster asset delivery. If you are interested in using this, please contact your dotCMS sales representative." +iconUrl: "https://static.dotcms.com/assets/icons/apps/dotCDN.png" +allowExtraParameters: false +params: + + cdnDomain: + label: "CDN domain" + value: "" + hidden: false + type: "STRING" + hint: "The base domain for your CDN, e.g. https://cdn.clientname.com" + required: true + + cdnApiKey: + label: "CDN API/Client Key" + value: "" + hidden: true + type: "STRING" + hint: "CDN API Key (dotCMS will provide)" + required: true + + cdnZoneId: + label: "CDN Zone ID" + value: "" + hidden: false + type: "STRING" + hint: "The machine ID for your CDN (dotCMS will provide)" + required: true diff --git a/dotCMS/src/main/webapp/WEB-INF/portlet.xml b/dotCMS/src/main/webapp/WEB-INF/portlet.xml index 859040355de0..9083c2320470 100644 --- a/dotCMS/src/main/webapp/WEB-INF/portlet.xml +++ b/dotCMS/src/main/webapp/WEB-INF/portlet.xml @@ -379,6 +379,12 @@ com.dotcms.rest.JSPPortlet + + dotCDN + dotCDN + com.dotcms.spring.portlet.PortletController + + diff --git a/dotCMS/src/main/webapp/WEB-INF/toolbox.xml b/dotCMS/src/main/webapp/WEB-INF/toolbox.xml index 23f8793935fc..991904a8fc36 100644 --- a/dotCMS/src/main/webapp/WEB-INF/toolbox.xml +++ b/dotCMS/src/main/webapp/WEB-INF/toolbox.xml @@ -74,6 +74,11 @@ request com.dotcms.rendering.velocity.viewtools.DotRenderTool + + dotcdn + request + com.dotcms.cdn.viewtool.DotCDNTool + import request diff --git a/dotCMS/src/test/java/com/dotcms/cdn/api/DotCDNAPITest.java b/dotCMS/src/test/java/com/dotcms/cdn/api/DotCDNAPITest.java new file mode 100644 index 000000000000..dd49df83d99b --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/cdn/api/DotCDNAPITest.java @@ -0,0 +1,326 @@ +package com.dotcms.cdn.api; + +import com.dotcms.cdn.CDNConstants; +import com.dotcms.http.CircuitBreakerUrl; +import com.dotcms.http.CircuitBreakerUrlBuilder; +import com.dotcms.rest.RestClientBuilder; +import com.dotcms.rest.exception.BadRequestException; +import com.dotcms.security.apps.AppSecrets; +import com.dotcms.security.apps.AppsAPI; +import com.dotcms.security.apps.Secret; +import com.dotcms.security.apps.Type; +import com.dotmarketing.beans.Host; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotSecurityException; +import org.apache.http.conn.ConnectTimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.MockitoAnnotations; + +import javax.ws.rs.client.Client; +import javax.ws.rs.client.Invocation; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +/** + * Unit tests for the DotCDNAPI class. These tests verify the behavior of the API methods + * related to fetching CDN stats and invalidating URLs using a mock setup. + */ +class DotCDNAPITest { + + private AutoCloseable closeable; + + @BeforeEach + void setUp() { + closeable = MockitoAnnotations.openMocks(this); + } + + @AfterEach + void tearDown() throws Exception { + closeable.close(); + } + + @Mock + private AppsAPI appsAPI; + + @Mock + private AppSecrets appSecrets; + + @Mock + private CircuitBreakerUrl circuitBreakerUrl; + + @Mock + private Host host; + + @Mock + private Client restClient; + + @Mock + private WebTarget webTarget; + + @Mock + private Invocation.Builder invocationBuilder; + + @Mock + private Response restResponse; + + private static final String TEST_API_KEY = "test-api-key"; + private static final String TEST_CDN_DOMAIN = "demo.dotcms.com"; + private static final String TEST_ZONE_ID = "1234567890"; + private static final String TEST_HOST_IDENTIFIER = "1234567890"; + + @Test + void test_getStats_Should_Return_Stats_For_Valid_Date_Range() + throws DotDataException, DotSecurityException, IOException { + + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + try (MockedConstruction ignoredUrlBuilder = + mockConstruction(CircuitBreakerUrlBuilder.class, + (urlBuilder, context) -> + setCircuitBreakerUrlBuilderMock(urlBuilder))) { + + final String statsResponse = Files.readString(Paths.get( + "src/test/resources/test-stats-response.json"), + StandardCharsets.UTF_8); + when(circuitBreakerUrl.doString()).thenReturn(statsResponse); + + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + final DotCDNStats dotCDNStats = dotCDNAPI.getStats("2023-10-01", "2023-10-02"); + assertNotNull(dotCDNStats); + } + } + } + + @Test + void test_getStats_Should_Throw_BadRequestException_For_Invalid_Dates() + throws DotDataException, DotSecurityException, IOException { + + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + try (MockedConstruction ignoredUrlBuilder = + mockConstruction(CircuitBreakerUrlBuilder.class, + (urlBuilder, context) -> + setCircuitBreakerUrlBuilderMock(urlBuilder))) { + + when(circuitBreakerUrl.doString()).thenThrow(new BadRequestException("Invalid date range")); + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + + assertThrows(BadRequestException.class, () -> + dotCDNAPI.getStats("2023-10-01", "2023-10-02") + ); + } + } + } + + @Test + void test_getStats_Should_Throw_Bad_Request_For_Timeout() + throws DotDataException, DotSecurityException, IOException { + + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + try (MockedConstruction ignoredUrlBuilder = + mockConstruction(CircuitBreakerUrlBuilder.class, + (urlBuilder, context) -> + setCircuitBreakerUrlBuilderMock(urlBuilder))) { + + when(circuitBreakerUrl.doString()).thenThrow(new ConnectTimeoutException("Request timed out")); + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + + assertThrows(BadRequestException.class, () -> + dotCDNAPI.getStats("2023-10-01", "2023-10-02") + ); + } + } + } + + @Test + void test_invalidate_Should_Return_True_For_Valid_Urls() + throws DotDataException, DotSecurityException, IOException { + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + @SuppressWarnings("unchecked") + final CircuitBreakerUrl.Response mockResponse = + org.mockito.Mockito.mock(CircuitBreakerUrl.Response.class); + when(mockResponse.getStatusCode()).thenReturn(200); + when(mockResponse.getResponse()).thenReturn(""); + + try (MockedConstruction ignoredUrlBuilder = + mockConstruction(CircuitBreakerUrlBuilder.class, + (urlBuilder, context) -> + setCircuitBreakerUrlBuilderMock(urlBuilder))) { + + when(circuitBreakerUrl.doResponse()).thenReturn(mockResponse); + + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + final boolean result = dotCDNAPI.invalidate(List.of( + "demo.dotcms.com/page1", "demo.dotcms.com/page2")); + + assertTrue(result); + } + } + } + + @Test + void test_invalidate_Should_Return_False_For_Bad_Request_Response() + throws DotDataException, DotSecurityException, IOException { + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + @SuppressWarnings("unchecked") + final CircuitBreakerUrl.Response mockResponse = + org.mockito.Mockito.mock(CircuitBreakerUrl.Response.class); + when(mockResponse.getStatusCode()).thenReturn(400); + when(mockResponse.getResponse()).thenReturn("Bad Request"); + + try (MockedConstruction ignoredUrlBuilder = + mockConstruction(CircuitBreakerUrlBuilder.class, + (urlBuilder, context) -> + setCircuitBreakerUrlBuilderMock(urlBuilder))) { + + when(circuitBreakerUrl.doResponse()).thenReturn(mockResponse); + + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + final boolean result = dotCDNAPI.invalidate(List.of( + "demo.dotcms.com/invalid-url-1" + )); + + assertFalse(result); + } + } + } + + @Test + void test_invalidateAll_Should_Return_True_For_Success() + throws DotDataException, DotSecurityException { + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + try (MockedStatic restClientBuilder = mockStatic(RestClientBuilder.class)) { + + setupRestClientMock(restClientBuilder); + when(restResponse.getStatus()).thenReturn(Response.Status.NO_CONTENT.getStatusCode()); + + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + final boolean result = dotCDNAPI.invalidateAll(); + + assertTrue(result); + } + } + } + + @Test + void test_invalidateAll_Should_Throw_Bad_Request_For_Invalid_Request() + throws DotDataException, DotSecurityException { + try (MockedStatic apiLocator = mockStatic(APILocator.class)) { + + setupAppsAPIMock(apiLocator); + setupCDNSecrets(); + when(host.getIdentifier()).thenReturn(TEST_HOST_IDENTIFIER); + + try (MockedStatic restClientBuilder = mockStatic(RestClientBuilder.class)) { + + setupRestClientMock(restClientBuilder); + when(restResponse.getStatus()).thenReturn(Response.Status.BAD_REQUEST.getStatusCode()); + + final DotCDNAPI dotCDNAPI = DotCDNAPI.api(host); + + assertThrows(BadRequestException.class, dotCDNAPI::invalidateAll); + } + } + } + + private void setCircuitBreakerUrlBuilderMock(CircuitBreakerUrlBuilder urlBuilder) { + when(urlBuilder.setHeaders(anyMap())).thenReturn(urlBuilder); + when(urlBuilder.setTimeout(anyLong())).thenReturn(urlBuilder); + when(urlBuilder.setUrl(anyString())).thenReturn(urlBuilder); + when(urlBuilder.setMethod(any())).thenReturn(urlBuilder); + when(urlBuilder.setThrowWhenError(anyBoolean())).thenReturn(urlBuilder); + when(urlBuilder.build()).thenReturn(circuitBreakerUrl); + } + + private void setupAppsAPIMock(final MockedStatic apiLocator) + throws DotDataException, DotSecurityException { + apiLocator.when(APILocator::getAppsAPI).thenReturn(appsAPI); + final Optional appSecretsResult = Optional.of(appSecrets); + when(appsAPI.getSecrets(eq(CDNConstants.DOT_CDN_APP_KEY), + anyBoolean(), any(), any())).thenReturn(appSecretsResult); + } + + private void setupCDNSecrets() { + final Secret apiKeySecret = createSecret(TEST_API_KEY); + final Secret cdnDomainSecret = createSecret(TEST_CDN_DOMAIN); + final Secret zoneIdSecret = createSecret(TEST_ZONE_ID); + + when(appSecrets.getSecrets()).thenReturn(Map.of( + CDNConstants.DOT_CDN_API_KEY, apiKeySecret, + CDNConstants.DOT_CDN_DOMAIN, cdnDomainSecret, + CDNConstants.DOT_CDN_ZONEID, zoneIdSecret + )); + } + + private Secret createSecret(String value) { + return Secret.builder() + .withValue(value) + .withHidden(false) + .withType(Type.STRING) + .build(); + } + + private void setupRestClientMock(MockedStatic restClientBuilder) { + restClientBuilder.when(RestClientBuilder::newClient).thenReturn(restClient); + when(restClient.target(anyString())).thenReturn(webTarget); + when(webTarget.request(any(MediaType.class))).thenReturn(invocationBuilder); + when(invocationBuilder.header(anyString(), anyString())).thenReturn(invocationBuilder); + when(invocationBuilder.post(any())).thenReturn(restResponse); + } +} diff --git a/dotCMS/src/test/resources/test-stats-response.json b/dotCMS/src/test/resources/test-stats-response.json new file mode 100644 index 000000000000..a234534abfe3 --- /dev/null +++ b/dotCMS/src/test/resources/test-stats-response.json @@ -0,0 +1,63 @@ +{ + "TotalBandwidthUsed": 4358868943127, + "TotalOriginTraffic": 201443350067, + "AverageOriginResponseTime": 107, + "OriginResponseTimeChart": { + "2025-04-21T00:00:00Z": 119.97915496496852, + "2025-04-22T00:00:00Z": 123.14402618107543 + }, + "TotalRequestsServed": 59506919, + "CacheHitRate": 92.00229640522979, + "BandwidthUsedChart": { + "2025-04-21T00:00:00Z": 323843276876, + "2025-04-22T00:00:00Z": 140979625395 + }, + "BandwidthCachedChart": { + "2025-04-21T00:00:00Z": 319176563428, + "2025-04-22T00:00:00Z": 139213074644 + }, + "CacheHitRateChart": { + "2025-04-21T00:00:00Z": 87.98853939047213, + "2025-04-22T00:00:00Z": 89.02157708669782 + }, + "RequestsServedChart": { + "2025-04-21T00:00:00Z": 4513198, + "2025-04-22T00:00:00Z": 1997582 + }, + "PullRequestsPulledChart": { + "2025-04-21T00:00:00Z": 542101, + "2025-04-22T00:00:00Z": 219303 + }, + "OriginShieldBandwidthUsedChart": { + "2025-04-21T00:00:00Z": 2655412083, + "2025-04-22T00:00:00Z": 1383082792 + }, + "OriginShieldInternalBandwidthUsedChart": { + "2025-04-21T00:00:00Z": 6216959163, + "2025-04-22T00:00:00Z": 3061534513 + }, + "OriginTrafficChart": { + "2025-04-21T00:00:00Z": 14447333558, + "2025-04-22T00:00:00Z": 7375473616 + }, + "UserBalanceHistoryChart": { + "2025-04-21T01:26:36": -657.07, + "2025-04-22T01:21:46": -688.27 + }, + "GeoTrafficDistribution": { + "NA: New York City, NY": 411752804365, + "NA: Miami, FL": 250155831412 + }, + "Error3xxChart": { + "2025-04-21T00:00:00Z": 0, + "2025-04-22T00:00:00Z": 0 + }, + "Error4xxChart": { + "2025-04-21T00:00:00Z": 0, + "2025-04-22T00:00:00Z": 0 + }, + "Error5xxChart": { + "2025-04-21T00:00:00Z": 0, + "2025-04-22T00:00:00Z": 0 + } +}