From 7ab2d0cf32359fd7074baf5689d1286d1e2dfac0 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 7 Jul 2026 14:27:22 -0600 Subject: [PATCH 1/7] Add timing print-outs to FAST.Farm --- CMakeLists.txt | 1 + glue-codes/fast-farm/CMakeLists.txt | 3 + glue-codes/fast-farm/src/FAST_Farm.f90 | 64 +- glue-codes/fast-farm/src/FAST_Farm_Subs.f90 | 658 +++++++++++++++++- modules/awae/CMakeLists.txt | 2 + modules/awae/src/AWAE.f90 | 193 ++++- modules/nwtc-library/src/SysFlangLinux.f90 | 2 +- modules/nwtc-library/src/SysGnuLinux.f90 | 2 +- modules/nwtc-library/src/SysGnuWin.f90 | 2 +- modules/nwtc-library/src/SysIFL.f90 | 2 +- modules/nwtc-library/src/SysIVF.f90 | 2 +- modules/nwtc-library/src/SysIVF_Labview.f90 | 2 +- .../nwtc-library/src/SysMatlabLinuxGnu.f90 | 2 +- .../nwtc-library/src/SysMatlabLinuxIntel.f90 | 2 +- modules/nwtc-library/src/SysMatlabWindows.f90 | 2 +- 15 files changed, 911 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d6bfa66fd..0cd03fa644 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ option(USE_DLL_INTERFACE "Enable runtime loading of dynamic libraries" on) option(FPE_TRAP_ENABLED "Enable FPE trap in compiler options" off) option(WIN_DLL_LOAD "Enable loading of Windows only DLL's (OrcaFlex, SoilDyn)" on) # This is mostly for testing purposes option(BUILD_FASTFARM "Enable building FAST.Farm" off) +option(FASTFARM_TIMING_PRINTS "Enable FAST.Farm timing instrumentation and summary output" off) option(BUILD_OPENFAST_CPP_API "Enable building OpenFAST - C++ API" off) option(BUILD_OPENFAST_CPP_DRIVER "Enable building OpenFAST C++ driver using C++ CFD API" off) option(BUILD_OPENFAST_LIB_DRIVER "Enable building OpenFAST driver using C++ Library API" off) diff --git a/glue-codes/fast-farm/CMakeLists.txt b/glue-codes/fast-farm/CMakeLists.txt index 96672632b6..80000952ca 100644 --- a/glue-codes/fast-farm/CMakeLists.txt +++ b/glue-codes/fast-farm/CMakeLists.txt @@ -29,6 +29,9 @@ add_executable(FAST.Farm src/FASTWrapper_Types.f90 ) target_link_libraries(FAST.Farm openfastlib_static wdlib awaelib) +if (FASTFARM_TIMING_PRINTS) + target_compile_definitions(FAST.Farm PRIVATE FF_TIMING_PRINTS) +endif() set_property(TARGET FAST.Farm PROPERTY LINKER_LANGUAGE Fortran) string(TOUPPER ${CMAKE_Fortran_COMPILER_ID} _compiler_id) diff --git a/glue-codes/fast-farm/src/FAST_Farm.f90 b/glue-codes/fast-farm/src/FAST_Farm.f90 index c3d63ed756..d10a161bfe 100644 --- a/glue-codes/fast-farm/src/FAST_Farm.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm.f90 @@ -53,6 +53,12 @@ PROGRAM FAST_Farm REAL(ReKi) :: SimStrtCPU !< User CPU time for simulation (without initialization) REAL(ReKi) :: ProgStrtCPU !< User CPU time for program (with initialization) +#ifdef FF_TIMING_PRINTS +REAL(DbKi) :: tmSer0 +REAL(DbKi) :: tmPar0 +REAL(DbKi) :: tmPar1 +#endif + ! these should probably go in the FAST.Farm registry: type(All_FastFarm_Data) :: farm @@ -74,6 +80,10 @@ PROGRAM FAST_Farm farm%p%NumTurbines = 0 t = 0 + +#ifdef FF_TIMING_PRINTS + call FarmTiming_Reset() +#endif InputFileName = "" ! make sure we don't think this is a "default" inputFileName if not specified on command line CALL CheckArgs( InputFileName, ErrStat=ErrStat, Flag=FlagArg ) @@ -101,8 +111,16 @@ PROGRAM FAST_Farm ! Initialization !............................................................................................................................... - call Farm_Initialize( farm, InputFileName, ErrStat, ErrMsg ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + call Farm_Initialize( farm, InputFileName, ErrStat, ErrMsg ) CALL CheckError( ErrStat, ErrMsg, 'during driver initialization' ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('INIT', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif !............................................................................................................................... ! Initial Calculate Output @@ -110,8 +128,16 @@ PROGRAM FAST_Farm CALL SimStatus_FirstTime( PrevSimTime, PrevClockTime, SimStrtTime, SimStrtCPU, t, farm%p%TMax ) - call FARM_InitialCO(farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + call FARM_InitialCO(farm, ErrStat, ErrMsg) CALL CheckError( ErrStat, ErrMsg, 'during initial calculate output' ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('ICO', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif END IF @@ -140,16 +166,32 @@ PROGRAM FAST_Farm t = n_t_global*farm%p%DT_low - CALL FARM_UpdateStates(t, n_t_global, farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + CALL FARM_UpdateStates(t, n_t_global, farm, ErrStat, ErrMsg) CALL CheckError( ErrStat, ErrMsg ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('US', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif t = (n_t_global+1)*farm%p%DT_low - CALL FARM_CalcOutput(t, farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + CALL FARM_CalcOutput(t, farm, ErrStat, ErrMsg) CALL CheckError( ErrStat, ErrMsg ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('CO', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif CALL SimStatus( PrevSimTime, PrevClockTime, t, farm%p%TMax ) @@ -160,12 +202,24 @@ PROGRAM FAST_Farm ! End: !............................................................................................................................... - call FARM_End(farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + call FARM_End(farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('END', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif ! Finalize AMReX library call amrex_finalize() CALL RunTimes( ProgStrtTime, ProgStrtCPU, SimStrtTime, SimStrtCPU, t ) + +#ifdef FF_TIMING_PRINTS + call FarmTiming_PrintSummary() +#endif call NormStop() CONTAINS diff --git a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 index 30a7160478..e2c37c4739 100644 --- a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 @@ -42,9 +42,255 @@ MODULE FAST_Farm_Subs IMPLICIT NONE integer(IntKi), private, parameter :: iED = 1 +#ifdef FF_TIMING_PRINTS + integer(IntKi), parameter :: FFTiming_MaxEntries = 256 + integer(IntKi), save :: FFTiming_Count = 0 + character(128), save :: FFTiming_Label(FFTiming_MaxEntries) + real(DbKi), save :: FFTiming_Serial(FFTiming_MaxEntries) = 0.0_DbKi + real(DbKi), save :: FFTiming_Par(FFTiming_MaxEntries) = 0.0_DbKi +#endif CONTAINS +subroutine FarmTiming_Reset() +#ifdef FF_TIMING_PRINTS + call AWAE_ResetTiming() + FFTiming_Count = 0 + FFTiming_Label = '' + FFTiming_Serial = 0.0_DbKi + FFTiming_Par = 0.0_DbKi +#endif +end subroutine FarmTiming_Reset + +#ifdef FF_TIMING_PRINTS +real(DbKi) function FarmTiming_WallTime() +#ifdef _OPENMP + FarmTiming_WallTime = omp_get_wtime() +#else + call cpu_time(FarmTiming_WallTime) +#endif +end function FarmTiming_WallTime + +subroutine FarmTiming_Add(label, serialDt, parDt) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialDt + real(DbKi), intent(in) :: parDt + integer(IntKi) :: i + + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(label)) then + FFTiming_Serial(i) = FFTiming_Serial(i) + serialDt + FFTiming_Par(i) = FFTiming_Par(i) + parDt + return + end if + end do + + if (FFTiming_Count < FFTiming_MaxEntries) then + FFTiming_Count = FFTiming_Count + 1 + FFTiming_Label(FFTiming_Count) = label + FFTiming_Serial(FFTiming_Count) = serialDt + FFTiming_Par(FFTiming_Count) = parDt + end if +end subroutine FarmTiming_Add + +subroutine FarmTiming_AddElapsed(label, serialStart, parStart) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialStart + real(DbKi), intent(in) :: parStart + real(DbKi) :: serialEnd, parEnd + + call cpu_time(serialEnd) + parEnd = FarmTiming_WallTime() + call FarmTiming_Add(label, serialEnd-serialStart, parEnd-parStart) +end subroutine FarmTiming_AddElapsed + +subroutine FarmTiming_DrainAWAEDetails() + integer(IntKi) :: n, i + character(128) :: labels(AWAE_MaxTimingEntries) + real(DbKi) :: serial(AWAE_MaxTimingEntries) + real(DbKi) :: par(AWAE_MaxTimingEntries) + + call AWAE_DrainTimingData(n, labels, serial, par) + do i = 1, n + call FarmTiming_Add(trim(labels(i)), serial(i), par(i)) + end do +end subroutine FarmTiming_DrainAWAEDetails + +logical function FarmTiming_IsStageChild(label, stage) + character(*), intent(in) :: label + character(*), intent(in) :: stage + character(24) :: prefix + integer(IntKi) :: nColons + + prefix = trim(stage)//':' + if (index(trim(label), trim(prefix)) /= 1) then + FarmTiming_IsStageChild = .false. + return + end if + + nColons = FarmTiming_CountColons(label) + FarmTiming_IsStageChild = (nColons == 2) +end function FarmTiming_IsStageChild + +integer(IntKi) function FarmTiming_CountColons(label) + character(*), intent(in) :: label + integer(IntKi) :: i + + FarmTiming_CountColons = 0 + do i = 1, len_trim(label) + if (label(i:i) == ':') FarmTiming_CountColons = FarmTiming_CountColons + 1 + end do +end function FarmTiming_CountColons + +logical function FarmTiming_IsParentChild(label, parent) + character(*), intent(in) :: label + character(*), intent(in) :: parent + character(128) :: prefix + integer(IntKi) :: parentColons, labelColons + + prefix = trim(parent)//':' + if (index(trim(label), trim(prefix)) /= 1) then + FarmTiming_IsParentChild = .false. + return + end if + + parentColons = FarmTiming_CountColons(parent) + labelColons = FarmTiming_CountColons(label) + FarmTiming_IsParentChild = (labelColons == parentColons + 1) +end function FarmTiming_IsParentChild + +subroutine FarmTiming_AddParentOthers(parent) + character(*), intent(in) :: parent + integer(IntKi) :: i, iParent + real(DbKi) :: serialChildren, parChildren + real(DbKi) :: serialOthers, parOthers + character(128) :: othersLabel + + iParent = 0 + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(parent)) then + iParent = i + exit + end if + end do + if (iParent == 0) return + + serialChildren = 0.0_DbKi + parChildren = 0.0_DbKi + do i = 1, FFTiming_Count + if (FarmTiming_IsParentChild(FFTiming_Label(i), parent)) then + serialChildren = serialChildren + FFTiming_Serial(i) + parChildren = parChildren + FFTiming_Par(i) + end if + end do + + serialOthers = FFTiming_Serial(iParent) - serialChildren + parOthers = FFTiming_Par(iParent) - parChildren + if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi + if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi + + othersLabel = trim(parent)//':Others' + call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) +end subroutine FarmTiming_AddParentOthers + +subroutine FarmTiming_AddStageOthers(stage) + character(*), intent(in) :: stage + integer(IntKi) :: i + integer(IntKi) :: iStage + real(DbKi) :: serialChildren, parChildren + real(DbKi) :: serialOthers, parOthers + character(32) :: othersLabel + + iStage = 0 + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(stage)) then + iStage = i + exit + end if + end do + if (iStage == 0) return + + serialChildren = 0.0_DbKi + parChildren = 0.0_DbKi + do i = 1, FFTiming_Count + if (FarmTiming_IsStageChild(FFTiming_Label(i), stage)) then + serialChildren = serialChildren + FFTiming_Serial(i) + parChildren = parChildren + FFTiming_Par(i) + end if + end do + + serialOthers = FFTiming_Serial(iStage) - serialChildren + parOthers = FFTiming_Par(iStage) - parChildren + if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi + if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi + + othersLabel = trim(stage)//':Others' + call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) +end subroutine FarmTiming_AddStageOthers + +subroutine FarmTiming_PrintLiveStart(stage, label, mode) + character(*), intent(in) :: stage + character(*), intent(in) :: label + character(*), intent(in) :: mode + character(384) :: line + + line = 'FFTiming|LIVE|Event=Start|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveStart + +subroutine FarmTiming_PrintLiveDone(stage, label, mode, dt) + character(*), intent(in) :: stage + character(*), intent(in) :: label + character(*), intent(in) :: mode + real(DbKi), intent(in) :: dt + character(384) :: line + + line = 'FFTiming|LIVE|Event=Done|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode)//'|Seconds='//trim(num2lstr(dt)) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveDone + +subroutine FarmTiming_PrintLiveThread(stage, label, itemId, threadId, dt) + character(*), intent(in) :: stage + character(*), intent(in) :: label + integer(IntKi), intent(in) :: itemId + integer(IntKi), intent(in) :: threadId + real(DbKi), intent(in) :: dt + character(384) :: line + + line = 'FFTiming|LIVE|Event=ThreadDone|Stage='//trim(stage)//'|Label='//trim(label)//'|Item='//trim(num2lstr(itemId))// & + '|Thread='//trim(num2lstr(threadId))//'|Seconds='//trim(num2lstr(dt)) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveThread +#endif + +subroutine FarmTiming_PrintSummary() +#ifdef FF_TIMING_PRINTS + integer(IntKi) :: i + character(384) :: line + + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput1') + call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput2') + call FarmTiming_AddParentOthers('US:AWAE:UpdateStates') + call FarmTiming_AddParentOthers('CO:AWAE:CalcOutput') + call FarmTiming_AddStageOthers('INIT') + call FarmTiming_AddStageOthers('ICO') + call FarmTiming_AddStageOthers('US') + call FarmTiming_AddStageOthers('CO') + call FarmTiming_AddStageOthers('END') + + call WrScr('') + call WrScr('FFTiming|SUMMARY|Begin') + call WrScr('FFTiming|SUMMARY|Columns=Label|CPU|Wall') + do i = 1, FFTiming_Count + line = 'FFTiming|SUMMARY|Label='//trim(FFTiming_Label(i))//'|CPU='//trim(num2lstr(FFTiming_Serial(i)))//'|Wall='//trim(num2lstr(FFTiming_Par(i))) + call WrScr(trim(line)) + end do + call WrScr('FFTiming|SUMMARY|End') + call WrScr('') +#endif +end subroutine FarmTiming_PrintSummary + subroutine TrilinearInterpRegGrid(V, pt, dims, val) real(SiKi), intent(in ) :: V(:,0:,0:,0:) !< The volume data being sampled @@ -153,10 +399,18 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) CHARACTER(*), PARAMETER :: RoutineName = 'Farm_Initialize' CHARACTER(ChanLen) :: OutList(Farm_MaxOutPts) ! list of user-requested output channels INTEGER(IntKi) :: i +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0, tmPar1, tmPar2 +#endif !.......... ErrStat = ErrID_None ErrMsg = "" AbortErrLev = ErrID_Fatal ! Until we read otherwise from the FAST input file, we abort only on FATAL errors + +#ifdef FF_TIMING_PRINTS + tmPar1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'Stage', 'Par') +#endif ! ... Open and read input files, initialize global parameters. ... @@ -242,9 +496,20 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) if (farm%p%WAT /= Mod_WAT_None .and. associated(farm%WAT_IfW%p%FlowField)) then AWAE_InitInput%WAT_FlowField => farm%WAT_IfW%p%FlowField endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'AWAE_Init', 'Par') + call AWAE_SetTimingStage('INIT') +#endif call AWAE_Init( AWAE_InitInput, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, farm%AWAE%OtherSt, farm%AWAE%y, & farm%AWAE%m, farm%p%DT_low, AWAE_InitOutput, ErrStat2, ErrMsg2 ) if(Failed()) return; +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:AWAE:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'AWAE_Init', 'Par', tmPar2-tmPar0) +#endif farm%AWAE%IsInitialized = .true. @@ -262,21 +527,51 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) !------------------- ! c. initialize WD (one instance per turbine, each can be done in parallel, too) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'WD_Init', 'Par') +#endif call Farm_InitWD( farm, WD_InitInput, ErrStat2, ErrMsg2 ); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:WD:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'WD_Init', 'Par', tmPar2-tmPar0) +#endif !............................................................................................................................... ! step 4: initialize FAST (each instance of FAST can also be done in parallel) !............................................................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'FWrap_Init', 'Par') +#endif CALL Farm_InitFAST( farm, WD_InitInput%InputFileData, AWAE_InitOutput, ErrStat2, ErrMsg2); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:FWrap:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'FWrap_Init', 'Par', tmPar2-tmPar0) +#endif !............................................................................................................................... ! step 4.5: initialize farm-level MoorDyn if applicable !............................................................................................................................... if (farm%p%MooringMod == 3) then +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'MD_Init', 'Serial') +#endif CALL Farm_InitMD( farm, ErrStat2, ErrMsg2); if(Failed()) return; ! FAST instances must be initialized first so that turbine initial positions are known +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:MD:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'MD_Init', 'Serial', tmPar2-tmPar0) +#endif end if !............................................................................................................................... @@ -284,19 +579,45 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) !............................................................................................................................... ! Set parameters for output channels: +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif CALL Farm_SetOutParam(OutList, farm, ErrStat2, ErrMsg2 ); if(Failed()) return; ! requires: p%NumOuts, sets: p%OutParam. +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('INIT:IO:SetOutParam', tmSer0, tmPar0) +#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_InitOutput( farm, ErrStat2, ErrMsg2 ); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('INIT:IO:InitOutput', tmSer0, tmPar0) +#endif ! Print the summary file if requested: IF (farm%p%SumPrint) THEN +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif CALL Farm_PrintSum( farm, WD_InitInput%InputFileData, ErrStat2, ErrMsg2 ); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('INIT:IO:PrintSummary', tmSer0, tmPar0) +#endif END IF !............................................................................................................................... ! Destroy initializion data !............................................................................................................................... CALL Cleanup() + +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('INIT', 'Stage', 'Par', tmPar2-tmPar1) +#endif CONTAINS SUBROUTINE Cleanup() @@ -1042,10 +1363,18 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) INTEGER(IntKi) :: ErrStat2 ! Temporary Error status CHARACTER(ErrMsgLen) :: ErrMsg2 ! Temporary Error message CHARACTER(*), PARAMETER :: RoutineName = 'FARM_InitialCO' +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0, tmPar1, tmPar2 +#endif ErrStat = ErrID_None ErrMsg = "" + +#ifdef FF_TIMING_PRINTS + tmPar1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'Stage', 'Par') +#endif @@ -1065,10 +1394,22 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) !-------------------- ! 1b. CALL AWAE_CO +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'AWAE_CalcOutput1', 'Par') + call AWAE_SetTimingStage('ICO:AWAE:CalcOutput1') +#endif call AWAE_CalcOutput( 0.0_DbKi, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('ICO:AWAE:CalcOutput1', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'AWAE_CalcOutput1', 'Par', tmPar2-tmPar0) +#endif !-------------------- ! 1c. transfer y_AWAE to u_F and u_WD @@ -1078,6 +1419,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) ! CALL F_t0 (can be done in parallel) !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'FWrap_t0', 'Par') +#endif DO nt = 1,farm%p%NumTurbines call FWrap_t0( farm%FWrap(nt)%u, farm%FWrap(nt)%p, farm%FWrap(nt)%x, farm%FWrap(nt)%xd, farm%FWrap(nt)%z, & @@ -1086,6 +1432,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) END DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('ICO:FWrap:t0', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'FWrap_t0', 'Par', tmPar2-tmPar0) +#endif !....................................................................................... ! Transfer y_F to u_WD @@ -1100,6 +1451,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) ! CALL WD_CO (can be done in parallel) !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'WD_CalcOutput', 'Par') +#endif DO nt = 1,farm%p%NumTurbines call WD_CalcOutput( 0.0_DbKi, farm%WD(nt)%u, farm%WD(nt)%p, farm%WD(nt)%x, farm%WD(nt)%xd, farm%WD(nt)%z, & @@ -1108,6 +1464,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) END DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('ICO:WD:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'WD_CalcOutput', 'Par', tmPar2-tmPar0) +#endif !....................................................................................... ! Transfer y_WD to u_AWAE @@ -1118,11 +1479,22 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) !....................................................................................... ! CALL AWAE_CO !....................................................................................... - +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'AWAE_CalcOutput2', 'Par') + call AWAE_SetTimingStage('ICO:AWAE:CalcOutput2') +#endif call AWAE_CalcOutput( 0.0_DbKi, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('ICO:AWAE:CalcOutput2', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'AWAE_CalcOutput2', 'Par', tmPar2-tmPar0) +#endif !....................................................................................... ! Transfer y_AWAE to u_F and u_WD @@ -1134,8 +1506,18 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) ! Write Output to File !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_WriteOutput(0, 0.0_DbKi, farm, ErrStat2, ErrMsg2) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) + +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('ICO:IO:WriteOutput', tmSer0, tmPar0) + tmPar2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('ICO', 'Stage', 'Par', tmPar2-tmPar1) +#endif end subroutine FARM_InitialCO !---------------------------------------------------------------------------------------------------------------------------------- @@ -1165,7 +1547,12 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) INTEGER(IntKi), ALLOCATABLE :: ErrStatF(:) ! Temporary Error status for FAST CHARACTER(ErrMsgLen), ALLOCATABLE :: ErrMsgF (:) ! Temporary Error message for FAST CHARACTER(*), PARAMETER :: RoutineName = 'FARM_UpdateStates' - REAL(DbKi) :: tm1,tm2,tm3, tm01, tm02, tm03, tmSF, tmSM ! timer variables + REAL(DbKi) :: tm1, tm2 +#ifdef FF_TIMING_PRINTS + REAL(DbKi) :: tmSer0, tmPar0, tmPar1 + REAL(DbKi), ALLOCATABLE :: fwrapThreadDt(:) + INTEGER(IntKi), ALLOCATABLE :: fwrapThreadId(:) +#endif ErrStat = ErrID_None ErrMsg = "" @@ -1175,6 +1562,11 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) allocate ( ErrMsgF ( farm%p%NumTurbines ), STAT=errStat2 ) if (errStat2 /= 0) call SetErrStat ( ErrID_Fatal, 'Could not allocate memory for ErrMsgF.', errStat, errMsg, RoutineName ) if (ErrStat >= AbortErrLev) return + +#ifdef FF_TIMING_PRINTS + tmPar1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'Stage', 'Par') +#endif @@ -1186,7 +1578,13 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) !-------------------- ! 1. CALL WD_US - + +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'WD_UpdateStates', 'Par') +#endif + !$OMP PARALLEL default(shared) !$OMP do private(nt, ErrStat2, ErrMsg2) schedule(runtime) DO nt = 1,farm%p%NumTurbines @@ -1205,6 +1603,12 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) END DO !$OMP END DO !$OMP END PARALLEL + +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('US:WD:UpdateStates', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'WD_UpdateStates', 'Par', tm2-tmPar0) +#endif if (ErrStat >= AbortErrLev) return @@ -1213,9 +1617,16 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! set the inputs needed for FAST (these are slow-varying so can just be done once per farm time step) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif do nt = 1,farm%p%NumTurbines call FWrap_SetWindTStart(farm%FWrap(nt)%u, farm%FWrap(nt)%m, t) end do +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('US:FWrap:SetWindTStart', tmSer0, tmPar0) +#endif !#ifdef printthreads @@ -1226,12 +1637,52 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! Original case: no shared moorings if (farm%p%MooringMod == 0) then +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'FWrap_Increment_NoMD', 'Par') + + allocate(fwrapThreadDt(farm%p%NumTurbines), fwrapThreadId(farm%p%NumTurbines), stat=ErrStat2) + if (ErrStat2 /= 0) then + call SetErrStat(ErrID_Fatal, 'Could not allocate memory for buffered FWrap thread timing output.', ErrStat, ErrMsg, RoutineName) + return + end if + fwrapThreadDt = -1.0_DbKi + fwrapThreadId = -1_IntKi +#endif + !$OMP PARALLEL DO DEFAULT(Shared) Private(nt) DO nt = 1,farm%p%NumTurbines +#ifdef FF_TIMING_PRINTS + fwrapThreadDt(nt) = FarmTiming_WallTime() +#endif call FWrap_Increment( t, n, farm%FWrap(nt)%u, farm%FWrap(nt)%p, farm%FWrap(nt)%x, farm%FWrap(nt)%xd, farm%FWrap(nt)%z, & farm%FWrap(nt)%OtherSt, farm%FWrap(nt)%y, farm%FWrap(nt)%m, ErrStatF(nt), ErrMsgF(nt) ) +#ifdef FF_TIMING_PRINTS + fwrapThreadDt(nt) = FarmTiming_WallTime() - fwrapThreadDt(nt) +#ifdef _OPENMP + fwrapThreadId(nt) = omp_get_thread_num() +#else + fwrapThreadId(nt) = 0_IntKi +#endif +#endif END DO !$OMP END PARALLEL DO + +#ifdef FF_TIMING_PRINTS + do nt = 1, farm%p%NumTurbines + if (fwrapThreadDt(nt) >= 0.0_DbKi) then + call FarmTiming_PrintLiveThread('US', 'FWrap_Increment_NoMD', nt, fwrapThreadId(nt), fwrapThreadDt(nt)) + end if + end do + deallocate(fwrapThreadDt, fwrapThreadId) +#endif + +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('US:FWrap:Increment', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'FWrap_Increment_NoMD', 'Par', tm2-tmPar0) +#endif ! Farm-level moorings case using MoorDyn else if (farm%p%MooringMod == 3) then @@ -1247,6 +1698,10 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! tm01 = omp_get_wtime() !#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif ! A nested parallel for loop to call each instance of OpenFAST in parallel !$OMP PARALLEL DO DEFAULT(Shared) Private(nt) DO nt = 1,farm%p%NumTurbines @@ -1254,14 +1709,26 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) farm%FWrap(nt)%OtherSt, farm%FWrap(nt)%y, farm%FWrap(nt)%m, ErrStatF(nt), ErrMsgF(nt) ) END DO !$OMP END PARALLEL DO + +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('US:FWrap:Increment', tmSer0, tmPar0) +#endif !#ifdef printthreads ! tm02 = omp_get_wtime() !#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif ! call farm-level MoorDyn time step here (can't multithread this with FAST since it needs inputs from all FAST instances) call Farm_MD_Increment( t2, n_FMD, farm, ErrStatMD, ErrMsgMD) call SetErrStat(ErrStatMD, ErrMsgMD, ErrStat, ErrMsg, 'FARM_UpdateStates') ! MD error status <<<<< + +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('US:MD:Increment', tmSer0, tmPar0) +#endif !#ifdef printthreads ! tm03 = omp_get_wtime() @@ -1285,6 +1752,27 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! write(*,*) 'Total FAST and Moordyn for FF_US took '//trim(num2lstr(tm2-tm1))//' seconds.' !#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'AWAE_UpdateStates', 'Par') + call AWAE_SetTimingStage('US:AWAE:UpdateStates') +#endif + call AWAE_UpdateStates( t, n, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & + farm%AWAE%OtherSt, farm%AWAE%m, ErrStatAWAE, ErrMsgAWAE ) + +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('US:AWAE:UpdateStates', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'AWAE_UpdateStates', 'Par', tm2-tmPar0) +#endif + + !#ifdef printthreads + ! tm3 = omp_get_wtime() + ! write(*,*) 'AWAE_US took '//trim(num2lstr(tm3-tm2))//' seconds.' + ! write(*,*) 'Total Farm_US took '//trim(num2lstr(tm3-tm1))//' seconds.' + !#endif ! update error messages from FAST's and AWAE's time steps DO nt = 1,farm%p%NumTurbines @@ -1292,14 +1780,30 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) END DO ! calculate outputs from FAST as needed by FAST.Farm +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'FWrap_CalcOutput', 'Serial') +#endif do nt = 1,farm%p%NumTurbines call FWrap_CalcOutput(farm%FWrap(nt)%p, farm%FWrap(nt)%u, farm%FWrap(nt)%y, farm%FWrap(nt)%m, ErrStat2, ErrMsg2) call setErrStat(ErrStat2,ErrMsg2,ErrStat,ErrMsg,RoutineName) end do +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('US:FWrap:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'FWrap_CalcOutput', 'Serial', tm2-tmPar0) +#endif + if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('US', 'Stage', 'Par', tm2-tmPar1) +#endif + end subroutine FARM_UpdateStates !---------------------------------------------------------------------------------------------------------------------------------- @@ -1587,7 +2091,10 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) CHARACTER(ErrMsgLen) :: ErrMsg2 ! Temporary Error message CHARACTER(*), PARAMETER :: RoutineName = 'FARM_CalcOutput' INTEGER(IntKi) :: n ! time step increment number -! REAL(DbKi) :: tm1 + REAL(DbKi) :: tm1, tm2 +#ifdef FF_TIMING_PRINTS + REAL(DbKi) :: tmSer0, tmPar0 +#endif ErrStat = ErrID_None ErrMsg = "" @@ -1602,7 +2109,15 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !....................................................................................... !-------------------- - ! 1. call WD_CO and transfer y_WD to u_AWAE + ! 1. call WD_CO and transfer y_WD to u_AWAE + +#ifdef FF_TIMING_PRINTS + tm1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'Stage', 'Serial') + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'WD_CalcOutput', 'Par') +#endif !$OMP PARALLEL DO DEFAULT (shared) PRIVATE(nt, ErrStat2, ErrMsg2) schedule(runtime) DO nt = 1,farm%p%NumTurbines @@ -1618,7 +2133,17 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !$OMP END PARALLEL DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('CO:WD:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'WD_CalcOutput', 'Par', tm2-tmPar0) +#endif + ! IO operation, not done using OpenMP +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif DO nt = 1,farm%p%NumTurbines call WD_WritePlaneOutputs( t, farm%WD(nt)%u, farm%WD(nt)%p, farm%WD(nt)%x, farm%WD(nt)%xd, farm%WD(nt)%z, & farm%WD(nt)%OtherSt, farm%WD(nt)%y, farm%WD(nt)%m, ErrStat2, ErrMsg2 ) @@ -1628,13 +2153,31 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) END DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:WD:WritePlaneOutputs', tmSer0, tmPar0) +#endif + +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Transfer_WD_to_AWAE(farm) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:Transfer:WD_to_AWAE', tmSer0, tmPar0) +#endif !-------------------- ! 2. Transfer y_F to u_WD - + +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Transfer_FAST_to_WD(farm) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:Transfer:FAST_to_WD', tmSer0, tmPar0) +#endif !....................................................................................... ! calculate AWAE outputs and perform rest of input-output solves @@ -1648,21 +2191,51 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !-------------------- ! 1. call AWAE_CO +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'AWAE_CalcOutput', 'Par') + call AWAE_SetTimingStage('CO:AWAE:CalcOutput') +#endif call AWAE_CalcOutput( t, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('CO:AWAE:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'AWAE_CalcOutput', 'Par', tm2-tmPar0) +#endif + !-------------------- ! 2. Transfer y_AWAE to u_F and u_WD +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Transfer_AWAE_to_WD(farm) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:Transfer:AWAE_to_WD', tmSer0, tmPar0) +#endif !....................................................................................... ! Write Output to File !....................................................................................... ! NOTE: Visualization data is output via the AWAE module +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + n = nint(t/farm%p%DT_low) call Farm_WriteOutput(n, t, farm, ErrStat2, ErrMsg2) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('CO:IO:WriteOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'WriteOutput', 'Serial', tm2-tmPar0) +#endif !....................................................................................... ! Write shared moorings visualization @@ -1689,6 +2262,12 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) endif endif +#ifdef FF_TIMING_PRINTS + call FarmTiming_Add('CO:IO:WriteMooringVis', 0.0_DbKi, 0.0_DbKi) + tm2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('CO', 'Stage', 'Serial', tm2-tm1) +#endif + ! write(*,*) 'Total Farm_CO-serial took '//trim(num2lstr(omp_get_wtime()-tm1))//' seconds.' end subroutine FARM_CalcOutput @@ -1710,11 +2289,18 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) INTEGER(IntKi) :: ErrStat2 ! Temporary Error status CHARACTER(ErrMsgLen) :: ErrMsg2 ! Temporary Error message CHARACTER(*), PARAMETER :: RoutineName = 'FARM_End' - - +#ifdef FF_TIMING_PRINTS + REAL(DbKi) :: tm1, tm2 + REAL(DbKi) :: tmSer0, tmPar0 +#endif ErrStat = ErrID_None ErrMsg = "" + +#ifdef FF_TIMING_PRINTS + tm1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('END', 'Stage', 'Serial') +#endif !....................................................................................... ! end all modules (1-4 can be done in parallel) @@ -1722,24 +2308,47 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) !-------------- ! 1. end AWAE +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (farm%WAT_IfW%IsInitialized) then call InflowWind_End(farm%WAT_IfW%u, farm%WAT_IfW%p, farm%WAT_IfW%x, farm%WAT_IfW%xd, farm%WAT_IfW%z, & farm%WAT_IfW%OtherSt, farm%WAT_IfW%y, farm%WAT_IfW%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) farm%WAT_IfW%IsInitialized = .false. endif +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:WAT:End', tmSer0, tmPar0) +#endif !-------------- ! 2. end AWAE +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('END', 'AWAE_End', 'Par') + call AWAE_SetTimingStage('END') +#endif if (farm%AWAE%IsInitialized) then call AWAE_End( farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) farm%AWAE%IsInitialized = .false. end if +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('END:AWAE:End', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('END', 'AWAE_End', 'Par', tm2-tmPar0) +#endif !-------------- ! 3. end WakeDynamics +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (allocated(farm%WD)) then DO nt = 1,farm%p%NumTurbines if (farm%WD(nt)%IsInitialized) then @@ -1750,9 +2359,16 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) end if END DO end if +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:WD:End', tmSer0, tmPar0) +#endif !-------------- ! 5. End each instance of FAST (each instance of FAST can be done in parallel, too) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (allocated(farm%FWrap)) then DO nt = 1,farm%p%NumTurbines if (farm%FWrap(nt)%IsInitialized) then @@ -1763,28 +2379,54 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) end if END DO end if +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:FWrap:End', tmSer0, tmPar0) +#endif !-------------- ! 6. End farm-level MoorDyn +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (farm%p%MooringMod == 3 .and. allocated(farm%MD%Input)) then call MD_End(farm%MD%Input(1), farm%MD%p, farm%MD%x, farm%MD%xd, farm%MD%z, farm%MD%OtherSt, farm%MD%y, farm%MD%m, ErrStat2, ErrMsg2) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName ) !TODO: any related items need to be cleared? end if +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:MD:End', tmSer0, tmPar0) +#endif !....................................................................................... ! close output file !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_EndOutput( farm, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg,RoutineName) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:IO:EndOutput', tmSer0, tmPar0) +#endif !....................................................................................... ! clear all data from 'farm' structure !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_DestroyAll_FastFarm_Data( farm, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('END:Farm:DestroyData', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('END', 'Stage', 'Serial', tm2-tm1) +#endif end subroutine FARM_End !---------------------------------------------------------------------------------------------------------------------------------- diff --git a/modules/awae/CMakeLists.txt b/modules/awae/CMakeLists.txt index fdb66b91b6..2a9f50b9b9 100644 --- a/modules/awae/CMakeLists.txt +++ b/modules/awae/CMakeLists.txt @@ -35,6 +35,8 @@ if (AMREX_READER) target_sources(awaelib_c PRIVATE src/amrex_utils.cpp) target_link_libraries(awaelib_c amrex_3d) target_compile_definitions(awaelib PRIVATE ENABLE_AMREX_LIB) +if (FASTFARM_TIMING_PRINTS) + target_compile_definitions(awaelib PRIVATE FF_TIMING_PRINTS) endif() install(TARGETS awaelib awaelib_c diff --git a/modules/awae/src/AWAE.f90 b/modules/awae/src/AWAE.f90 index 0f8360cc54..853a992b8b 100644 --- a/modules/awae/src/AWAE.f90 +++ b/modules/awae/src/AWAE.f90 @@ -48,6 +48,20 @@ module AWAE ! continuous states, and updating discrete states public :: AWAE_CalcOutput ! Routine for computing outputs public :: AWAE_CalcConstrStateResidual ! Tight coupling routine for returning the constraint state residual + public :: AWAE_ResetTiming ! Reset compile-time timing accumulators + public :: AWAE_GetTimingData ! Retrieve compile-time timing accumulators + public :: AWAE_DrainTimingData ! Retrieve and reset compile-time timing accumulators + public :: AWAE_SetTimingStage ! Set timing label prefix used by AWAE_CalcOutput timing + + integer(IntKi), parameter, public :: AWAE_MaxTimingEntries = 32 + +#ifdef FF_TIMING_PRINTS + integer(IntKi), save :: AWAE_TimingCount = 0 + character(128), save :: AWAE_TimingLabel(AWAE_MaxTimingEntries) + real(DbKi), save :: AWAE_TimingSerial(AWAE_MaxTimingEntries) = 0.0_DbKi + real(DbKi), save :: AWAE_TimingPar(AWAE_MaxTimingEntries) = 0.0_DbKi + character(64), save :: AWAE_TimingPrefix = '' +#endif ! Unit testing routines @@ -58,6 +72,108 @@ module AWAE contains +subroutine AWAE_SetTimingStage(stage) + character(*), intent(in) :: stage +#ifdef FF_TIMING_PRINTS + AWAE_TimingPrefix = stage +#endif +end subroutine AWAE_SetTimingStage + +subroutine AWAE_ResetTiming() +#ifdef FF_TIMING_PRINTS + AWAE_TimingCount = 0 + AWAE_TimingLabel = '' + AWAE_TimingSerial = 0.0_DbKi + AWAE_TimingPar = 0.0_DbKi + AWAE_TimingPrefix = '' +#endif +end subroutine AWAE_ResetTiming + +subroutine AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) + integer(IntKi), intent(out) :: numEntries + character(*), intent(out) :: labels(:) + real(DbKi), intent(out) :: serialTimes(:) + real(DbKi), intent(out) :: parTimes(:) + integer(IntKi) :: n, i + + numEntries = 0 + labels = '' + serialTimes = 0.0_DbKi + parTimes = 0.0_DbKi + +#ifdef FF_TIMING_PRINTS + n = min(AWAE_TimingCount, min(size(labels), min(size(serialTimes), size(parTimes)))) + numEntries = n + do i = 1, n + labels(i) = AWAE_TimingLabel(i) + serialTimes(i) = AWAE_TimingSerial(i) + parTimes(i) = AWAE_TimingPar(i) + end do +#endif +end subroutine AWAE_GetTimingData + +subroutine AWAE_DrainTimingData(numEntries, labels, serialTimes, parTimes) + integer(IntKi), intent(out) :: numEntries + character(*), intent(out) :: labels(:) + real(DbKi), intent(out) :: serialTimes(:) + real(DbKi), intent(out) :: parTimes(:) + + call AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) +#ifdef FF_TIMING_PRINTS + AWAE_TimingCount = 0 + AWAE_TimingLabel = '' + AWAE_TimingSerial = 0.0_DbKi + AWAE_TimingPar = 0.0_DbKi +#endif +end subroutine AWAE_DrainTimingData + +#ifdef FF_TIMING_PRINTS +real(DbKi) function AWAE_WallTime() +#ifdef _OPENMP + AWAE_WallTime = omp_get_wtime() +#else + call cpu_time(AWAE_WallTime) +#endif +end function AWAE_WallTime + +subroutine AWAE_AddTiming(label, serialDt, parDt) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialDt + real(DbKi), intent(in) :: parDt + integer(IntKi) :: i + + do i = 1, AWAE_TimingCount + if (trim(AWAE_TimingLabel(i)) == trim(label)) then + AWAE_TimingSerial(i) = AWAE_TimingSerial(i) + serialDt + AWAE_TimingPar(i) = AWAE_TimingPar(i) + parDt + return + end if + end do + + if (AWAE_TimingCount < AWAE_MaxTimingEntries) then + AWAE_TimingCount = AWAE_TimingCount + 1 + AWAE_TimingLabel(AWAE_TimingCount) = label + AWAE_TimingSerial(AWAE_TimingCount) = serialDt + AWAE_TimingPar(AWAE_TimingCount) = parDt + end if +end subroutine AWAE_AddTiming + +subroutine AWAE_AddStageTiming(action, serialStart, parStart) + character(*), intent(in) :: action + real(DbKi), intent(in) :: serialStart + real(DbKi), intent(in) :: parStart + real(DbKi) :: serialEnd, parEnd + character(128) :: label + + if (len_trim(AWAE_TimingPrefix) == 0) return + + call cpu_time(serialEnd) + parEnd = AWAE_WallTime() + label = trim(AWAE_TimingPrefix)//':'//trim(action) + call AWAE_AddTiming(label, serialEnd-serialStart, parEnd-parStart) +end subroutine AWAE_AddStageTiming +#endif + subroutine ExtractSlice( sliceType, s, s0, szs, sz1, sz2, ds, V, slice) @@ -1724,10 +1840,16 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) real(ReKi), allocatable :: AccUVW(:,:) logical :: WriteWindVTK real(DbKi) :: t - +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0 +#endif + errStat = ErrID_None errMsg = "" - + +#ifdef FF_TIMING_PRINTS + if (len_trim(AWAE_TimingPrefix) == 0) AWAE_TimingPrefix = 'US:AWAE:UpdateStates' +#endif ! If last time step, don't populate high-resolution grid if (n == (p%NumDT - 1)) then n_high_low = 0 @@ -1742,6 +1864,11 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) ! Populate low resolution grids based on ambient wind source !---------------------------------------------------------------------------- +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + select case (p%Mod_AmbWind) ! File-based ambient wind @@ -1772,10 +1899,19 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) end select +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('PopulateLowRes', tmSer0, tmPar0) +#endif + !---------------------------------------------------------------------------- ! Populate high-resolution grid based on ambient wind source !---------------------------------------------------------------------------- +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + select case (p%Mod_AmbWind) ! File-based ambient wind @@ -1897,12 +2033,21 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) end select +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('PopulateHighRes', tmSer0, tmPar0) +#endif + !---------------------------------------------------------------------------- ! Propagate WAT tracer !---------------------------------------------------------------------------- if (p%WAT_Enabled) then + #ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() + #endif + ! Find mean velocity of all turbine disks xd%Ufarm = 0.0_ReKi do nt=1,p%NumTurbines @@ -1912,6 +2057,10 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) ! add mean velocity * dt to the tracer for the position of the WAT box xd%WAT_B_Box = xd%WAT_B_Box + xd%Ufarm*real(p%dt_low,ReKi) + + #ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('PropagateWAT', tmSer0, tmPar0) + #endif endif contains @@ -1957,6 +2106,10 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg CHARACTER(1024) :: FileName INTEGER(IntKi) :: Un ! unit number of opened file logical :: WriteWindVTK +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0 +#endif + logical :: WriteWindVTK errStat = ErrID_None errMsg = "" @@ -1964,7 +2117,14 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg ! some variables and indexing n = nint(t / p%dt_low) n_high = n*p%n_high_low - call ComputeLocals(n, u, p, y, m, ErrStat2, ErrMsg2); if (Failed()) return; +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + call ComputeLocals(n, u, p, y, m, errStat2, errMsg2); if (Failed()) return; +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('ComputeLocals', tmSer0, tmPar0) +#endif ! Set flag to write wind VTK files if it's the correct step WriteWindVTK = mod(n, p%WrDisSkp1) == 0 @@ -1977,16 +2137,33 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg call CalcWakePointTurbineGridInteractions(p, m, u) - ! High-resolution grid output - call HighResGridCalcOutput(n_high, u, p, xd, y, m, ErrStat2, ErrMsg2) - if (Failed()) return + ! high-res +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + call HighResGridCalcOutput(n_high, u, p, xd, y, m, errStat2, errMsg2); if (Failed()) return; +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('HighResGridCalcOutput', tmSer0, tmPar0) +#endif ! Low-resolution grid output +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif call LowResGridCalcOutput(n, u, p, xd, y, m, ErrStat2, ErrMsg2) if (Failed()) return +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('LowResGridCalcOutput', tmSer0, tmPar0) +#endif ! If it's time to write wind VTK files if (WriteWindVTK) then + #ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() + #endif if (p%WrDisWind) then call WriteDisWindFiles(n, p%WrDisSkp1, p, y, m, ErrStat2, ErrMsg2) @@ -2046,6 +2223,10 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg p%LowRes%dXYZ, m%outVizXZPlane, ErrStat2, ErrMsg2) if (Failed()) return end do + + #ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('WriteDisWind', tmSer0, tmPar0) + #endif end if contains diff --git a/modules/nwtc-library/src/SysFlangLinux.f90 b/modules/nwtc-library/src/SysFlangLinux.f90 index 621e405528..8805951090 100644 --- a/modules/nwtc-library/src/SysFlangLinux.f90 +++ b/modules/nwtc-library/src/SysFlangLinux.f90 @@ -56,7 +56,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console. Unit 6 causes ADAMS to crash. - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Linux' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysGnuLinux.f90 b/modules/nwtc-library/src/SysGnuLinux.f90 index 0fb51921b3..6eddb9b410 100644 --- a/modules/nwtc-library/src/SysGnuLinux.f90 +++ b/modules/nwtc-library/src/SysGnuLinux.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Linux' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysGnuWin.f90 b/modules/nwtc-library/src/SysGnuWin.f90 index 95794cf195..1422989cb2 100644 --- a/modules/nwtc-library/src/SysGnuWin.f90 +++ b/modules/nwtc-library/src/SysGnuWin.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Windows' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysIFL.f90 b/modules/nwtc-library/src/SysIFL.f90 index f2ccbe1cde..bf427e8fb6 100644 --- a/modules/nwtc-library/src/SysIFL.f90 +++ b/modules/nwtc-library/src/SysIFL.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'Intel Fortran for Linux' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysIVF.f90 b/modules/nwtc-library/src/SysIVF.f90 index d0e7227657..41af20386a 100644 --- a/modules/nwtc-library/src/SysIVF.f90 +++ b/modules/nwtc-library/src/SysIVF.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 7 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}]; Note: NewLine change to ACHAR(10) here on Windows to fix issues with C/Fortran interoperability using WrScr CHARACTER(*), PARAMETER :: OS_Desc = 'Intel Visual Fortran for Windows'! Description of the language/OS diff --git a/modules/nwtc-library/src/SysIVF_Labview.f90 b/modules/nwtc-library/src/SysIVF_Labview.f90 index ac42539000..51a57144f5 100644 --- a/modules/nwtc-library/src/SysIVF_Labview.f90 +++ b/modules/nwtc-library/src/SysIVF_Labview.f90 @@ -74,7 +74,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 7 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. diff --git a/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 b/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 index 6b2a847d24..1072ca8e3b 100644 --- a/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 +++ b/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 @@ -60,7 +60,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Linux with Matlab' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 b/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 index a31dc15cad..303206a9a4 100644 --- a/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 +++ b/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 @@ -60,7 +60,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'Intel Fortran for Linux with Matlab' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysMatlabWindows.f90 b/modules/nwtc-library/src/SysMatlabWindows.f90 index c462d92df9..417a0e9ef6 100644 --- a/modules/nwtc-library/src/SysMatlabWindows.f90 +++ b/modules/nwtc-library/src/SysMatlabWindows.f90 @@ -48,7 +48,7 @@ MODULE SysSubs INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. From a7c3f7f06694ae9a3fa290fd55ab785ad7de7777 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 7 Jul 2026 14:57:25 -0600 Subject: [PATCH 2/7] Fix issues found by copilot --- modules/awae/CMakeLists.txt | 2 ++ modules/awae/src/AWAE.f90 | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/awae/CMakeLists.txt b/modules/awae/CMakeLists.txt index 2a9f50b9b9..1c91099c87 100644 --- a/modules/awae/CMakeLists.txt +++ b/modules/awae/CMakeLists.txt @@ -35,6 +35,8 @@ if (AMREX_READER) target_sources(awaelib_c PRIVATE src/amrex_utils.cpp) target_link_libraries(awaelib_c amrex_3d) target_compile_definitions(awaelib PRIVATE ENABLE_AMREX_LIB) +endif() + if (FASTFARM_TIMING_PRINTS) target_compile_definitions(awaelib PRIVATE FF_TIMING_PRINTS) endif() diff --git a/modules/awae/src/AWAE.f90 b/modules/awae/src/AWAE.f90 index 853a992b8b..50eccc4086 100644 --- a/modules/awae/src/AWAE.f90 +++ b/modules/awae/src/AWAE.f90 @@ -2109,7 +2109,6 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg #ifdef FF_TIMING_PRINTS real(DbKi) :: tmSer0, tmPar0 #endif - logical :: WriteWindVTK errStat = ErrID_None errMsg = "" From 78fddd9c098565d732f7ad4c94e723aea82ae9c2 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 7 Jul 2026 14:58:03 -0600 Subject: [PATCH 3/7] FAST.Farm timing. Adding some missing timings. --- glue-codes/fast-farm/src/FAST_Farm.f90 | 16 +++++++++++++ glue-codes/fast-farm/src/FAST_Farm_Subs.f90 | 18 +++++++++++++-- modules/awae/src/AWAE.f90 | 25 +++++++++++++-------- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/glue-codes/fast-farm/src/FAST_Farm.f90 b/glue-codes/fast-farm/src/FAST_Farm.f90 index d10a161bfe..4bf59344ed 100644 --- a/glue-codes/fast-farm/src/FAST_Farm.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm.f90 @@ -95,7 +95,15 @@ PROGRAM FAST_Farm endif ! Initialize AMReX library +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call amrex_init(arg_parmparse=.false.) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('AMREX:Init', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif CALL FAST_ProgStart( Farm_Ver ) ! put this after CheckArgs because CheckArgs assumes we haven't called this routine, yet. @@ -213,7 +221,15 @@ PROGRAM FAST_Farm #endif ! Finalize AMReX library +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call amrex_finalize() +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('AMREX:Finalize', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif CALL RunTimes( ProgStrtTime, ProgStrtCPU, SimStrtTime, SimStrtCPU, t ) diff --git a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 index e2c37c4739..8e0775fcfe 100644 --- a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 @@ -272,6 +272,7 @@ subroutine FarmTiming_PrintSummary() call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput1') call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput2') call FarmTiming_AddParentOthers('US:AWAE:UpdateStates') + call FarmTiming_AddParentOthers('CO:AWAE:UpdateStates') call FarmTiming_AddParentOthers('CO:AWAE:CalcOutput') call FarmTiming_AddStageOthers('INIT') call FarmTiming_AddStageOthers('ICO') @@ -1540,7 +1541,7 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) INTEGER(IntKi) :: n_ss INTEGER(IntKi) :: n_FMD REAL(DbKi) :: t2 ! time within the FAST-MoorDyn substepping loop for shared moorings - INTEGER(IntKi) :: ErrStatMD, ErrStat2 + INTEGER(IntKi) :: ErrStatMD, ErrStatAWAE, ErrStat2 CHARACTER(ErrMsgLen) :: ErrMsg2 CHARACTER(ErrMsgLen) :: ErrMsgAWAE CHARACTER(ErrMsgLen) :: ErrMsgMD @@ -1758,8 +1759,9 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) call FarmTiming_PrintLiveStart('US', 'AWAE_UpdateStates', 'Par') call AWAE_SetTimingStage('US:AWAE:UpdateStates') #endif - call AWAE_UpdateStates( t, n, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & + call AWAE_UpdateStates( n, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%m, ErrStatAWAE, ErrMsgAWAE ) + call SetErrStat(ErrStatAWAE, ErrMsgAWAE, ErrStat, ErrMsg, 'FARM_UpdateStates') ! AWAE error status #ifdef FF_TIMING_PRINTS tm2 = FarmTiming_WallTime() @@ -2185,9 +2187,21 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !-------------------- ! 0. call AWAE_UpdateStates to get the ambient wind and calculate wake-grid interactions +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'AWAE_UpdateStates', 'Par') + call AWAE_SetTimingStage('CO:AWAE:UpdateStates') +#endif call AWAE_UpdateStates( n, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('CO:AWAE:UpdateStates', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'AWAE_UpdateStates', 'Par', tm2-tmPar0) +#endif !-------------------- ! 1. call AWAE_CO diff --git a/modules/awae/src/AWAE.f90 b/modules/awae/src/AWAE.f90 index 50eccc4086..6430a11603 100644 --- a/modules/awae/src/AWAE.f90 +++ b/modules/awae/src/AWAE.f90 @@ -2043,10 +2043,10 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) if (p%WAT_Enabled) then - #ifdef FF_TIMING_PRINTS +#ifdef FF_TIMING_PRINTS call cpu_time(tmSer0) tmPar0 = AWAE_WallTime() - #endif +#endif ! Find mean velocity of all turbine disks xd%Ufarm = 0.0_ReKi @@ -2058,9 +2058,9 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) ! add mean velocity * dt to the tracer for the position of the WAT box xd%WAT_B_Box = xd%WAT_B_Box + xd%Ufarm*real(p%dt_low,ReKi) - #ifdef FF_TIMING_PRINTS +#ifdef FF_TIMING_PRINTS call AWAE_AddStageTiming('PropagateWAT', tmSer0, tmPar0) - #endif +#endif endif contains @@ -2134,7 +2134,14 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg ! m%iPlaneTurbTurb(2,p%NumTurbines,p%NumTurbines) (High-res grid) !---------------------------------------------------------------------------- +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif call CalcWakePointTurbineGridInteractions(p, m, u) +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('CalcWakePointTurbineGridInteractions', tmSer0, tmPar0) +#endif ! high-res #ifdef FF_TIMING_PRINTS @@ -2159,10 +2166,10 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg ! If it's time to write wind VTK files if (WriteWindVTK) then - #ifdef FF_TIMING_PRINTS +#ifdef FF_TIMING_PRINTS call cpu_time(tmSer0) tmPar0 = AWAE_WallTime() - #endif +#endif if (p%WrDisWind) then call WriteDisWindFiles(n, p%WrDisSkp1, p, y, m, ErrStat2, ErrMsg2) @@ -2223,9 +2230,9 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg if (Failed()) return end do - #ifdef FF_TIMING_PRINTS - call AWAE_AddStageTiming('WriteDisWind', tmSer0, tmPar0) - #endif +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('WriteDisWind', tmSer0, tmPar0) +#endif end if contains From 47c09169654eb9f7d265c742d206a4e3039d93e4 Mon Sep 17 00:00:00 2001 From: Regis Thedin Date: Tue, 7 Jul 2026 15:43:34 -0600 Subject: [PATCH 4/7] Fix mooring viz timing --- glue-codes/fast-farm/src/FAST_Farm_Subs.f90 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 index 8e0775fcfe..d99cb8a91e 100644 --- a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 @@ -2254,6 +2254,10 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !....................................................................................... ! Write shared moorings visualization !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif ! Write visualization meshes if (farm%p%MooringMod == 3) then @@ -2277,8 +2281,9 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) endif #ifdef FF_TIMING_PRINTS - call FarmTiming_Add('CO:IO:WriteMooringVis', 0.0_DbKi, 0.0_DbKi) tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('CO:IO:WriteMooringVis', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'WriteMooringVis', 'Serial', tm2-tmPar0) call FarmTiming_PrintLiveDone('CO', 'Stage', 'Serial', tm2-tm1) #endif From 93d891531f80e28e3d6f116b9f899a322a5b94d9 Mon Sep 17 00:00:00 2001 From: andrew-platt Date: Wed, 8 Jul 2026 15:04:37 -0600 Subject: [PATCH 5/7] Refactor: move FarmTiming subroutines to dedicated module Extract all FarmTiming_* subroutines and module variables from FAST_Farm_Subs.f90 into a new FAST_Farm_timings.f90 module. The new file has no #ifdef FF_TIMING_PRINTS guards (it is conditionally compiled via CMakeLists.txt). FAST_Farm_Subs conditionally USEs the module. Co-authored-by: Claude Opus 4.6 Co-authored-by: GitHub Copilot --- glue-codes/fast-farm/CMakeLists.txt | 1 + glue-codes/fast-farm/src/FAST_Farm_Subs.f90 | 250 +--------------- .../fast-farm/src/FAST_Farm_timings.f90 | 273 ++++++++++++++++++ 3 files changed, 277 insertions(+), 247 deletions(-) create mode 100644 glue-codes/fast-farm/src/FAST_Farm_timings.f90 diff --git a/glue-codes/fast-farm/CMakeLists.txt b/glue-codes/fast-farm/CMakeLists.txt index 80000952ca..c86bdbff59 100644 --- a/glue-codes/fast-farm/CMakeLists.txt +++ b/glue-codes/fast-farm/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(FAST.Farm ) target_link_libraries(FAST.Farm openfastlib_static wdlib awaelib) if (FASTFARM_TIMING_PRINTS) + target_sources(FAST.Farm PRIVATE src/FAST_Farm_timings.f90) target_compile_definitions(FAST.Farm PRIVATE FF_TIMING_PRINTS) endif() set_property(TARGET FAST.Farm PROPERTY LINKER_LANGUAGE Fortran) diff --git a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 index d99cb8a91e..4dbcc246bb 100644 --- a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 @@ -38,260 +38,16 @@ MODULE FAST_Farm_Subs #ifdef _OPENMP USE OMP_LIB #endif +#ifdef FF_TIMING_PRINTS + USE FAST_Farm_Timings +#endif IMPLICIT NONE integer(IntKi), private, parameter :: iED = 1 -#ifdef FF_TIMING_PRINTS - integer(IntKi), parameter :: FFTiming_MaxEntries = 256 - integer(IntKi), save :: FFTiming_Count = 0 - character(128), save :: FFTiming_Label(FFTiming_MaxEntries) - real(DbKi), save :: FFTiming_Serial(FFTiming_MaxEntries) = 0.0_DbKi - real(DbKi), save :: FFTiming_Par(FFTiming_MaxEntries) = 0.0_DbKi -#endif CONTAINS -subroutine FarmTiming_Reset() -#ifdef FF_TIMING_PRINTS - call AWAE_ResetTiming() - FFTiming_Count = 0 - FFTiming_Label = '' - FFTiming_Serial = 0.0_DbKi - FFTiming_Par = 0.0_DbKi -#endif -end subroutine FarmTiming_Reset - -#ifdef FF_TIMING_PRINTS -real(DbKi) function FarmTiming_WallTime() -#ifdef _OPENMP - FarmTiming_WallTime = omp_get_wtime() -#else - call cpu_time(FarmTiming_WallTime) -#endif -end function FarmTiming_WallTime - -subroutine FarmTiming_Add(label, serialDt, parDt) - character(*), intent(in) :: label - real(DbKi), intent(in) :: serialDt - real(DbKi), intent(in) :: parDt - integer(IntKi) :: i - - do i = 1, FFTiming_Count - if (trim(FFTiming_Label(i)) == trim(label)) then - FFTiming_Serial(i) = FFTiming_Serial(i) + serialDt - FFTiming_Par(i) = FFTiming_Par(i) + parDt - return - end if - end do - - if (FFTiming_Count < FFTiming_MaxEntries) then - FFTiming_Count = FFTiming_Count + 1 - FFTiming_Label(FFTiming_Count) = label - FFTiming_Serial(FFTiming_Count) = serialDt - FFTiming_Par(FFTiming_Count) = parDt - end if -end subroutine FarmTiming_Add - -subroutine FarmTiming_AddElapsed(label, serialStart, parStart) - character(*), intent(in) :: label - real(DbKi), intent(in) :: serialStart - real(DbKi), intent(in) :: parStart - real(DbKi) :: serialEnd, parEnd - - call cpu_time(serialEnd) - parEnd = FarmTiming_WallTime() - call FarmTiming_Add(label, serialEnd-serialStart, parEnd-parStart) -end subroutine FarmTiming_AddElapsed - -subroutine FarmTiming_DrainAWAEDetails() - integer(IntKi) :: n, i - character(128) :: labels(AWAE_MaxTimingEntries) - real(DbKi) :: serial(AWAE_MaxTimingEntries) - real(DbKi) :: par(AWAE_MaxTimingEntries) - - call AWAE_DrainTimingData(n, labels, serial, par) - do i = 1, n - call FarmTiming_Add(trim(labels(i)), serial(i), par(i)) - end do -end subroutine FarmTiming_DrainAWAEDetails - -logical function FarmTiming_IsStageChild(label, stage) - character(*), intent(in) :: label - character(*), intent(in) :: stage - character(24) :: prefix - integer(IntKi) :: nColons - - prefix = trim(stage)//':' - if (index(trim(label), trim(prefix)) /= 1) then - FarmTiming_IsStageChild = .false. - return - end if - - nColons = FarmTiming_CountColons(label) - FarmTiming_IsStageChild = (nColons == 2) -end function FarmTiming_IsStageChild - -integer(IntKi) function FarmTiming_CountColons(label) - character(*), intent(in) :: label - integer(IntKi) :: i - - FarmTiming_CountColons = 0 - do i = 1, len_trim(label) - if (label(i:i) == ':') FarmTiming_CountColons = FarmTiming_CountColons + 1 - end do -end function FarmTiming_CountColons - -logical function FarmTiming_IsParentChild(label, parent) - character(*), intent(in) :: label - character(*), intent(in) :: parent - character(128) :: prefix - integer(IntKi) :: parentColons, labelColons - - prefix = trim(parent)//':' - if (index(trim(label), trim(prefix)) /= 1) then - FarmTiming_IsParentChild = .false. - return - end if - - parentColons = FarmTiming_CountColons(parent) - labelColons = FarmTiming_CountColons(label) - FarmTiming_IsParentChild = (labelColons == parentColons + 1) -end function FarmTiming_IsParentChild - -subroutine FarmTiming_AddParentOthers(parent) - character(*), intent(in) :: parent - integer(IntKi) :: i, iParent - real(DbKi) :: serialChildren, parChildren - real(DbKi) :: serialOthers, parOthers - character(128) :: othersLabel - - iParent = 0 - do i = 1, FFTiming_Count - if (trim(FFTiming_Label(i)) == trim(parent)) then - iParent = i - exit - end if - end do - if (iParent == 0) return - - serialChildren = 0.0_DbKi - parChildren = 0.0_DbKi - do i = 1, FFTiming_Count - if (FarmTiming_IsParentChild(FFTiming_Label(i), parent)) then - serialChildren = serialChildren + FFTiming_Serial(i) - parChildren = parChildren + FFTiming_Par(i) - end if - end do - - serialOthers = FFTiming_Serial(iParent) - serialChildren - parOthers = FFTiming_Par(iParent) - parChildren - if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi - if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi - - othersLabel = trim(parent)//':Others' - call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) -end subroutine FarmTiming_AddParentOthers - -subroutine FarmTiming_AddStageOthers(stage) - character(*), intent(in) :: stage - integer(IntKi) :: i - integer(IntKi) :: iStage - real(DbKi) :: serialChildren, parChildren - real(DbKi) :: serialOthers, parOthers - character(32) :: othersLabel - - iStage = 0 - do i = 1, FFTiming_Count - if (trim(FFTiming_Label(i)) == trim(stage)) then - iStage = i - exit - end if - end do - if (iStage == 0) return - - serialChildren = 0.0_DbKi - parChildren = 0.0_DbKi - do i = 1, FFTiming_Count - if (FarmTiming_IsStageChild(FFTiming_Label(i), stage)) then - serialChildren = serialChildren + FFTiming_Serial(i) - parChildren = parChildren + FFTiming_Par(i) - end if - end do - - serialOthers = FFTiming_Serial(iStage) - serialChildren - parOthers = FFTiming_Par(iStage) - parChildren - if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi - if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi - - othersLabel = trim(stage)//':Others' - call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) -end subroutine FarmTiming_AddStageOthers - -subroutine FarmTiming_PrintLiveStart(stage, label, mode) - character(*), intent(in) :: stage - character(*), intent(in) :: label - character(*), intent(in) :: mode - character(384) :: line - - line = 'FFTiming|LIVE|Event=Start|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode) - call WrScr(trim(line)) -end subroutine FarmTiming_PrintLiveStart - -subroutine FarmTiming_PrintLiveDone(stage, label, mode, dt) - character(*), intent(in) :: stage - character(*), intent(in) :: label - character(*), intent(in) :: mode - real(DbKi), intent(in) :: dt - character(384) :: line - - line = 'FFTiming|LIVE|Event=Done|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode)//'|Seconds='//trim(num2lstr(dt)) - call WrScr(trim(line)) -end subroutine FarmTiming_PrintLiveDone - -subroutine FarmTiming_PrintLiveThread(stage, label, itemId, threadId, dt) - character(*), intent(in) :: stage - character(*), intent(in) :: label - integer(IntKi), intent(in) :: itemId - integer(IntKi), intent(in) :: threadId - real(DbKi), intent(in) :: dt - character(384) :: line - - line = 'FFTiming|LIVE|Event=ThreadDone|Stage='//trim(stage)//'|Label='//trim(label)//'|Item='//trim(num2lstr(itemId))// & - '|Thread='//trim(num2lstr(threadId))//'|Seconds='//trim(num2lstr(dt)) - call WrScr(trim(line)) -end subroutine FarmTiming_PrintLiveThread -#endif - -subroutine FarmTiming_PrintSummary() -#ifdef FF_TIMING_PRINTS - integer(IntKi) :: i - character(384) :: line - - call FarmTiming_DrainAWAEDetails() - call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput1') - call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput2') - call FarmTiming_AddParentOthers('US:AWAE:UpdateStates') - call FarmTiming_AddParentOthers('CO:AWAE:UpdateStates') - call FarmTiming_AddParentOthers('CO:AWAE:CalcOutput') - call FarmTiming_AddStageOthers('INIT') - call FarmTiming_AddStageOthers('ICO') - call FarmTiming_AddStageOthers('US') - call FarmTiming_AddStageOthers('CO') - call FarmTiming_AddStageOthers('END') - - call WrScr('') - call WrScr('FFTiming|SUMMARY|Begin') - call WrScr('FFTiming|SUMMARY|Columns=Label|CPU|Wall') - do i = 1, FFTiming_Count - line = 'FFTiming|SUMMARY|Label='//trim(FFTiming_Label(i))//'|CPU='//trim(num2lstr(FFTiming_Serial(i)))//'|Wall='//trim(num2lstr(FFTiming_Par(i))) - call WrScr(trim(line)) - end do - call WrScr('FFTiming|SUMMARY|End') - call WrScr('') -#endif -end subroutine FarmTiming_PrintSummary - subroutine TrilinearInterpRegGrid(V, pt, dims, val) real(SiKi), intent(in ) :: V(:,0:,0:,0:) !< The volume data being sampled diff --git a/glue-codes/fast-farm/src/FAST_Farm_timings.f90 b/glue-codes/fast-farm/src/FAST_Farm_timings.f90 new file mode 100644 index 0000000000..95e859325f --- /dev/null +++ b/glue-codes/fast-farm/src/FAST_Farm_timings.f90 @@ -0,0 +1,273 @@ +!********************************************************************************************************************************** +! LICENSING +! Copyright (C) 2026 National Renewable Energy Laboratory +! +! This file is part of FAST.Farm. +! +! Licensed under the Apache License, Version 2.0 (the "License"); +! you may not use this file except in compliance with the License. +! You may obtain a copy of the License at +! +! http://www.apache.org/licenses/LICENSE-2.0 +! +! Unless required by applicable law or agreed to in writing, software +! distributed under the License is distributed on an "AS IS" BASIS, +! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +! See the License for the specific language governing permissions and +! limitations under the License. +! +!********************************************************************************************************************************** +MODULE FAST_Farm_Timings + + USE NWTC_Library + USE AWAE_Timings, only: AWAE_ResetTiming, AWAE_DrainTimingData, AWAE_MaxTimingEntries + +#ifdef _OPENMP + USE OMP_LIB +#endif + + IMPLICIT NONE + + integer(IntKi), parameter :: FFTiming_MaxEntries = 256 + integer(IntKi), save :: FFTiming_Count = 0 + character(128), save :: FFTiming_Label(FFTiming_MaxEntries) + real(DbKi), save :: FFTiming_Serial(FFTiming_MaxEntries) = 0.0_DbKi + real(DbKi), save :: FFTiming_Par(FFTiming_MaxEntries) = 0.0_DbKi + +CONTAINS + +subroutine FarmTiming_Reset() + call AWAE_ResetTiming() + FFTiming_Count = 0 + FFTiming_Label = '' + FFTiming_Serial = 0.0_DbKi + FFTiming_Par = 0.0_DbKi +end subroutine FarmTiming_Reset + +real(DbKi) function FarmTiming_WallTime() +#ifdef _OPENMP + FarmTiming_WallTime = omp_get_wtime() +#else + call cpu_time(FarmTiming_WallTime) +#endif +end function FarmTiming_WallTime + +subroutine FarmTiming_Add(label, serialDt, parDt) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialDt + real(DbKi), intent(in) :: parDt + integer(IntKi) :: i + + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(label)) then + FFTiming_Serial(i) = FFTiming_Serial(i) + serialDt + FFTiming_Par(i) = FFTiming_Par(i) + parDt + return + end if + end do + + if (FFTiming_Count < FFTiming_MaxEntries) then + FFTiming_Count = FFTiming_Count + 1 + FFTiming_Label(FFTiming_Count) = label + FFTiming_Serial(FFTiming_Count) = serialDt + FFTiming_Par(FFTiming_Count) = parDt + end if +end subroutine FarmTiming_Add + +subroutine FarmTiming_AddElapsed(label, serialStart, parStart) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialStart + real(DbKi), intent(in) :: parStart + real(DbKi) :: serialEnd, parEnd + + call cpu_time(serialEnd) + parEnd = FarmTiming_WallTime() + call FarmTiming_Add(label, serialEnd-serialStart, parEnd-parStart) +end subroutine FarmTiming_AddElapsed + +subroutine FarmTiming_DrainAWAEDetails() + integer(IntKi) :: n, i + character(128) :: labels(AWAE_MaxTimingEntries) + real(DbKi) :: serial(AWAE_MaxTimingEntries) + real(DbKi) :: par(AWAE_MaxTimingEntries) + + call AWAE_DrainTimingData(n, labels, serial, par) + do i = 1, n + call FarmTiming_Add(trim(labels(i)), serial(i), par(i)) + end do +end subroutine FarmTiming_DrainAWAEDetails + +logical function FarmTiming_IsStageChild(label, stage) + character(*), intent(in) :: label + character(*), intent(in) :: stage + character(24) :: prefix + integer(IntKi) :: nColons + + prefix = trim(stage)//':' + if (index(trim(label), trim(prefix)) /= 1) then + FarmTiming_IsStageChild = .false. + return + end if + + nColons = FarmTiming_CountColons(label) + FarmTiming_IsStageChild = (nColons == 2) +end function FarmTiming_IsStageChild + +integer(IntKi) function FarmTiming_CountColons(label) + character(*), intent(in) :: label + integer(IntKi) :: i + + FarmTiming_CountColons = 0 + do i = 1, len_trim(label) + if (label(i:i) == ':') FarmTiming_CountColons = FarmTiming_CountColons + 1 + end do +end function FarmTiming_CountColons + +logical function FarmTiming_IsParentChild(label, parent) + character(*), intent(in) :: label + character(*), intent(in) :: parent + character(128) :: prefix + integer(IntKi) :: parentColons, labelColons + + prefix = trim(parent)//':' + if (index(trim(label), trim(prefix)) /= 1) then + FarmTiming_IsParentChild = .false. + return + end if + + parentColons = FarmTiming_CountColons(parent) + labelColons = FarmTiming_CountColons(label) + FarmTiming_IsParentChild = (labelColons == parentColons + 1) +end function FarmTiming_IsParentChild + +subroutine FarmTiming_AddParentOthers(parent) + character(*), intent(in) :: parent + integer(IntKi) :: i, iParent + real(DbKi) :: serialChildren, parChildren + real(DbKi) :: serialOthers, parOthers + character(128) :: othersLabel + + iParent = 0 + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(parent)) then + iParent = i + exit + end if + end do + if (iParent == 0) return + + serialChildren = 0.0_DbKi + parChildren = 0.0_DbKi + do i = 1, FFTiming_Count + if (FarmTiming_IsParentChild(FFTiming_Label(i), parent)) then + serialChildren = serialChildren + FFTiming_Serial(i) + parChildren = parChildren + FFTiming_Par(i) + end if + end do + + serialOthers = FFTiming_Serial(iParent) - serialChildren + parOthers = FFTiming_Par(iParent) - parChildren + if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi + if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi + + othersLabel = trim(parent)//':Others' + call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) +end subroutine FarmTiming_AddParentOthers + +subroutine FarmTiming_AddStageOthers(stage) + character(*), intent(in) :: stage + integer(IntKi) :: i + integer(IntKi) :: iStage + real(DbKi) :: serialChildren, parChildren + real(DbKi) :: serialOthers, parOthers + character(32) :: othersLabel + + iStage = 0 + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(stage)) then + iStage = i + exit + end if + end do + if (iStage == 0) return + + serialChildren = 0.0_DbKi + parChildren = 0.0_DbKi + do i = 1, FFTiming_Count + if (FarmTiming_IsStageChild(FFTiming_Label(i), stage)) then + serialChildren = serialChildren + FFTiming_Serial(i) + parChildren = parChildren + FFTiming_Par(i) + end if + end do + + serialOthers = FFTiming_Serial(iStage) - serialChildren + parOthers = FFTiming_Par(iStage) - parChildren + if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi + if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi + + othersLabel = trim(stage)//':Others' + call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) +end subroutine FarmTiming_AddStageOthers + +subroutine FarmTiming_PrintLiveStart(stage, label, mode) + character(*), intent(in) :: stage + character(*), intent(in) :: label + character(*), intent(in) :: mode + character(384) :: line + + line = 'FFTiming|LIVE|Event=Start|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveStart + +subroutine FarmTiming_PrintLiveDone(stage, label, mode, dt) + character(*), intent(in) :: stage + character(*), intent(in) :: label + character(*), intent(in) :: mode + real(DbKi), intent(in) :: dt + character(384) :: line + + line = 'FFTiming|LIVE|Event=Done|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode)//'|Seconds='//trim(num2lstr(dt)) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveDone + +subroutine FarmTiming_PrintLiveThread(stage, label, itemId, threadId, dt) + character(*), intent(in) :: stage + character(*), intent(in) :: label + integer(IntKi), intent(in) :: itemId + integer(IntKi), intent(in) :: threadId + real(DbKi), intent(in) :: dt + character(384) :: line + + line = 'FFTiming|LIVE|Event=ThreadDone|Stage='//trim(stage)//'|Label='//trim(label)//'|Item='//trim(num2lstr(itemId))// & + '|Thread='//trim(num2lstr(threadId))//'|Seconds='//trim(num2lstr(dt)) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveThread + +subroutine FarmTiming_PrintSummary() + integer(IntKi) :: i + character(384) :: line + + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput1') + call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput2') + call FarmTiming_AddParentOthers('US:AWAE:UpdateStates') + call FarmTiming_AddParentOthers('CO:AWAE:UpdateStates') + call FarmTiming_AddParentOthers('CO:AWAE:CalcOutput') + call FarmTiming_AddStageOthers('INIT') + call FarmTiming_AddStageOthers('ICO') + call FarmTiming_AddStageOthers('US') + call FarmTiming_AddStageOthers('CO') + call FarmTiming_AddStageOthers('END') + + call WrScr('') + call WrScr('FFTiming|SUMMARY|Begin') + call WrScr('FFTiming|SUMMARY|Columns=Label|CPU|Wall') + do i = 1, FFTiming_Count + line = 'FFTiming|SUMMARY|Label='//trim(FFTiming_Label(i))//'|CPU='//trim(num2lstr(FFTiming_Serial(i)))//'|Wall='//trim(num2lstr(FFTiming_Par(i))) + call WrScr(trim(line)) + end do + call WrScr('FFTiming|SUMMARY|End') + call WrScr('') +end subroutine FarmTiming_PrintSummary + +END MODULE FAST_Farm_Timings From 85ef199e506bae31215981944674b487a17b0a16 Mon Sep 17 00:00:00 2001 From: andrew-platt Date: Wed, 8 Jul 2026 20:14:51 -0600 Subject: [PATCH 6/7] Refactor: move AWAE timing subroutines to dedicated module Extract all AWAE timing subroutines and module variables from AWAE.f90 into a new AWAE_timings.f90 module. The new file has no #ifdef FF_TIMING_PRINTS guards (it is conditionally compiled via CMakeLists.txt). AWAE conditionally USEs the module and re-exports the public symbols. Co-authored-by: Claude Opus 4.6 Co-authored-by: GitHub Copilot --- modules/awae/CMakeLists.txt | 1 + modules/awae/src/AWAE.f90 | 120 ++------------------------ modules/awae/src/AWAE_timings.f90 | 136 ++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 111 deletions(-) create mode 100644 modules/awae/src/AWAE_timings.f90 diff --git a/modules/awae/CMakeLists.txt b/modules/awae/CMakeLists.txt index 1c91099c87..2079ef67a0 100644 --- a/modules/awae/CMakeLists.txt +++ b/modules/awae/CMakeLists.txt @@ -38,6 +38,7 @@ if (AMREX_READER) endif() if (FASTFARM_TIMING_PRINTS) + target_sources(awaelib PRIVATE src/AWAE_timings.f90) target_compile_definitions(awaelib PRIVATE FF_TIMING_PRINTS) endif() diff --git a/modules/awae/src/AWAE.f90 b/modules/awae/src/AWAE.f90 index 6430a11603..8b329cd772 100644 --- a/modules/awae/src/AWAE.f90 +++ b/modules/awae/src/AWAE.f90 @@ -34,6 +34,9 @@ module AWAE #ifdef _OPENMP use OMP_LIB #endif +#ifdef FF_TIMING_PRINTS + use AWAE_Timings +#endif implicit none @@ -48,19 +51,16 @@ module AWAE ! continuous states, and updating discrete states public :: AWAE_CalcOutput ! Routine for computing outputs public :: AWAE_CalcConstrStateResidual ! Tight coupling routine for returning the constraint state residual +#ifdef FF_TIMING_PRINTS + ! exposing the AWAE_timings module public subroutines for use in FAST.Farm public :: AWAE_ResetTiming ! Reset compile-time timing accumulators public :: AWAE_GetTimingData ! Retrieve compile-time timing accumulators public :: AWAE_DrainTimingData ! Retrieve and reset compile-time timing accumulators public :: AWAE_SetTimingStage ! Set timing label prefix used by AWAE_CalcOutput timing - - integer(IntKi), parameter, public :: AWAE_MaxTimingEntries = 32 - -#ifdef FF_TIMING_PRINTS - integer(IntKi), save :: AWAE_TimingCount = 0 - character(128), save :: AWAE_TimingLabel(AWAE_MaxTimingEntries) - real(DbKi), save :: AWAE_TimingSerial(AWAE_MaxTimingEntries) = 0.0_DbKi - real(DbKi), save :: AWAE_TimingPar(AWAE_MaxTimingEntries) = 0.0_DbKi - character(64), save :: AWAE_TimingPrefix = '' + public :: AWAE_MaxTimingEntries + public :: AWAE_WallTime + public :: AWAE_AddTiming + public :: AWAE_AddStageTiming #endif @@ -72,108 +72,6 @@ module AWAE contains -subroutine AWAE_SetTimingStage(stage) - character(*), intent(in) :: stage -#ifdef FF_TIMING_PRINTS - AWAE_TimingPrefix = stage -#endif -end subroutine AWAE_SetTimingStage - -subroutine AWAE_ResetTiming() -#ifdef FF_TIMING_PRINTS - AWAE_TimingCount = 0 - AWAE_TimingLabel = '' - AWAE_TimingSerial = 0.0_DbKi - AWAE_TimingPar = 0.0_DbKi - AWAE_TimingPrefix = '' -#endif -end subroutine AWAE_ResetTiming - -subroutine AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) - integer(IntKi), intent(out) :: numEntries - character(*), intent(out) :: labels(:) - real(DbKi), intent(out) :: serialTimes(:) - real(DbKi), intent(out) :: parTimes(:) - integer(IntKi) :: n, i - - numEntries = 0 - labels = '' - serialTimes = 0.0_DbKi - parTimes = 0.0_DbKi - -#ifdef FF_TIMING_PRINTS - n = min(AWAE_TimingCount, min(size(labels), min(size(serialTimes), size(parTimes)))) - numEntries = n - do i = 1, n - labels(i) = AWAE_TimingLabel(i) - serialTimes(i) = AWAE_TimingSerial(i) - parTimes(i) = AWAE_TimingPar(i) - end do -#endif -end subroutine AWAE_GetTimingData - -subroutine AWAE_DrainTimingData(numEntries, labels, serialTimes, parTimes) - integer(IntKi), intent(out) :: numEntries - character(*), intent(out) :: labels(:) - real(DbKi), intent(out) :: serialTimes(:) - real(DbKi), intent(out) :: parTimes(:) - - call AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) -#ifdef FF_TIMING_PRINTS - AWAE_TimingCount = 0 - AWAE_TimingLabel = '' - AWAE_TimingSerial = 0.0_DbKi - AWAE_TimingPar = 0.0_DbKi -#endif -end subroutine AWAE_DrainTimingData - -#ifdef FF_TIMING_PRINTS -real(DbKi) function AWAE_WallTime() -#ifdef _OPENMP - AWAE_WallTime = omp_get_wtime() -#else - call cpu_time(AWAE_WallTime) -#endif -end function AWAE_WallTime - -subroutine AWAE_AddTiming(label, serialDt, parDt) - character(*), intent(in) :: label - real(DbKi), intent(in) :: serialDt - real(DbKi), intent(in) :: parDt - integer(IntKi) :: i - - do i = 1, AWAE_TimingCount - if (trim(AWAE_TimingLabel(i)) == trim(label)) then - AWAE_TimingSerial(i) = AWAE_TimingSerial(i) + serialDt - AWAE_TimingPar(i) = AWAE_TimingPar(i) + parDt - return - end if - end do - - if (AWAE_TimingCount < AWAE_MaxTimingEntries) then - AWAE_TimingCount = AWAE_TimingCount + 1 - AWAE_TimingLabel(AWAE_TimingCount) = label - AWAE_TimingSerial(AWAE_TimingCount) = serialDt - AWAE_TimingPar(AWAE_TimingCount) = parDt - end if -end subroutine AWAE_AddTiming - -subroutine AWAE_AddStageTiming(action, serialStart, parStart) - character(*), intent(in) :: action - real(DbKi), intent(in) :: serialStart - real(DbKi), intent(in) :: parStart - real(DbKi) :: serialEnd, parEnd - character(128) :: label - - if (len_trim(AWAE_TimingPrefix) == 0) return - - call cpu_time(serialEnd) - parEnd = AWAE_WallTime() - label = trim(AWAE_TimingPrefix)//':'//trim(action) - call AWAE_AddTiming(label, serialEnd-serialStart, parEnd-parStart) -end subroutine AWAE_AddStageTiming -#endif - subroutine ExtractSlice( sliceType, s, s0, szs, sz1, sz2, ds, V, slice) diff --git a/modules/awae/src/AWAE_timings.f90 b/modules/awae/src/AWAE_timings.f90 new file mode 100644 index 0000000000..f654d5330a --- /dev/null +++ b/modules/awae/src/AWAE_timings.f90 @@ -0,0 +1,136 @@ +!********************************************************************************************************************************** +! LICENSING +! Copyright (C) 2026 National Renewable Energy Laboratory +! +! This file is part of Ambient Wind and Array Effects model for FAST.Farm. +! +! Licensed under the Apache License, Version 2.0 (the "License"); +! you may not use this file except in compliance with the License. +! You may obtain a copy of the License at +! +! http://www.apache.org/licenses/LICENSE-2.0 +! +! Unless required by applicable law or agreed to in writing, software +! distributed under the License is distributed on an "AS IS" BASIS, +! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +! See the License for the specific language governing permissions and +! limitations under the License. +! +!********************************************************************************************************************************** +MODULE AWAE_Timings + + USE NWTC_Library + +#ifdef _OPENMP + USE OMP_LIB +#endif + + IMPLICIT NONE + + integer(IntKi), parameter, public :: AWAE_MaxTimingEntries = 32 + + ! Typically we dissalow the `save` attribute for module variables, but we make an exception here since + ! these variables are used to store timing data that is collected across multiple subroutine calls, but + ! only outside of OMP loops. If we were to allow OMP loops to call these subroutines, then we would + ! need to use threadprivate variables instead of save variables. + integer(IntKi), save :: AWAE_TimingCount = 0 + character(128), save :: AWAE_TimingLabel(AWAE_MaxTimingEntries) + real(DbKi), save :: AWAE_TimingSerial(AWAE_MaxTimingEntries) = 0.0_DbKi + real(DbKi), save :: AWAE_TimingPar(AWAE_MaxTimingEntries) = 0.0_DbKi + character(64), save :: AWAE_TimingPrefix = '' + +CONTAINS + +subroutine AWAE_SetTimingStage(stage) + character(*), intent(in) :: stage + AWAE_TimingPrefix = stage +end subroutine AWAE_SetTimingStage + +subroutine AWAE_ResetTiming() + AWAE_TimingCount = 0 + AWAE_TimingLabel = '' + AWAE_TimingSerial = 0.0_DbKi + AWAE_TimingPar = 0.0_DbKi + AWAE_TimingPrefix = '' +end subroutine AWAE_ResetTiming + +subroutine AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) + integer(IntKi), intent(out) :: numEntries + character(*), intent(out) :: labels(:) + real(DbKi), intent(out) :: serialTimes(:) + real(DbKi), intent(out) :: parTimes(:) + integer(IntKi) :: n, i + + numEntries = 0 + labels = '' + serialTimes = 0.0_DbKi + parTimes = 0.0_DbKi + + n = min(AWAE_TimingCount, min(size(labels), min(size(serialTimes), size(parTimes)))) + numEntries = n + do i = 1, n + labels(i) = AWAE_TimingLabel(i) + serialTimes(i) = AWAE_TimingSerial(i) + parTimes(i) = AWAE_TimingPar(i) + end do +end subroutine AWAE_GetTimingData + +subroutine AWAE_DrainTimingData(numEntries, labels, serialTimes, parTimes) + integer(IntKi), intent(out) :: numEntries + character(*), intent(out) :: labels(:) + real(DbKi), intent(out) :: serialTimes(:) + real(DbKi), intent(out) :: parTimes(:) + + call AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) + AWAE_TimingCount = 0 + AWAE_TimingLabel = '' + AWAE_TimingSerial = 0.0_DbKi + AWAE_TimingPar = 0.0_DbKi +end subroutine AWAE_DrainTimingData + +real(DbKi) function AWAE_WallTime() +#ifdef _OPENMP + AWAE_WallTime = omp_get_wtime() +#else + call cpu_time(AWAE_WallTime) +#endif +end function AWAE_WallTime + +subroutine AWAE_AddTiming(label, serialDt, parDt) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialDt + real(DbKi), intent(in) :: parDt + integer(IntKi) :: i + + do i = 1, AWAE_TimingCount + if (trim(AWAE_TimingLabel(i)) == trim(label)) then + AWAE_TimingSerial(i) = AWAE_TimingSerial(i) + serialDt + AWAE_TimingPar(i) = AWAE_TimingPar(i) + parDt + return + end if + end do + + if (AWAE_TimingCount < AWAE_MaxTimingEntries) then + AWAE_TimingCount = AWAE_TimingCount + 1 + AWAE_TimingLabel(AWAE_TimingCount) = label + AWAE_TimingSerial(AWAE_TimingCount) = serialDt + AWAE_TimingPar(AWAE_TimingCount) = parDt + end if +end subroutine AWAE_AddTiming + +subroutine AWAE_AddStageTiming(action, serialStart, parStart) + character(*), intent(in) :: action + real(DbKi), intent(in) :: serialStart + real(DbKi), intent(in) :: parStart + real(DbKi) :: serialEnd, parEnd + character(128) :: label + + if (len_trim(AWAE_TimingPrefix) == 0) return + + call cpu_time(serialEnd) + parEnd = AWAE_WallTime() + label = trim(AWAE_TimingPrefix)//':'//trim(action) + call AWAE_AddTiming(label, serialEnd-serialStart, parEnd-parStart) +end subroutine AWAE_AddStageTiming + +END MODULE AWAE_Timings From 731de7803f67f0056529c409737f18ee303432f0 Mon Sep 17 00:00:00 2001 From: andrew-platt Date: Wed, 8 Jul 2026 20:23:00 -0600 Subject: [PATCH 7/7] Fix WrScr format overflow for MaxWrScrLen > 99 Expand the indent format specifier in WrScr from I2 to I3 (and Frm from CHARACTER(10) to CHARACTER(11)) to support indent values up to 999. With MaxWrScrLen increased to 256, the previous I2 format could overflow for indent > 99, producing invalid format strings. Fixes issue raised in: https://github.com/OpenFAST/openfast/pull/3378/changes#r3539678079 Co-authored-by: Claude Opus 4.6 Co-authored-by: GitHub Copilot --- modules/nwtc-library/src/NWTC_IO.f90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nwtc-library/src/NWTC_IO.f90 b/modules/nwtc-library/src/NWTC_IO.f90 index 1fbe24a6cc..11779b7a12 100644 --- a/modules/nwtc-library/src/NWTC_IO.f90 +++ b/modules/nwtc-library/src/NWTC_IO.f90 @@ -7845,7 +7845,7 @@ RECURSIVE SUBROUTINE WrScr ( InStr ) INTEGER :: MaxLen ! Maximum number of columns to be written to the screen. INTEGER :: NewLineIndx ! The string index where the NewLine character occurs - CHARACTER(10) :: Frm ! Format specifier for the output. + CHARACTER(11) :: Frm ! Format specifier for the output. CHARACTER(Len=:),allocatable :: Str ! The next string to be processed @@ -7872,8 +7872,8 @@ RECURSIVE SUBROUTINE WrScr ( InStr ) Indent = MIN( Indent, MaxLen-2 ) ! at least 2 characters per line MaxLen = MaxLen - Indent IF ( Indent > 0 ) THEN - Frm = '(1X, X,A)' - WRITE (Frm(5:6),'(I2)') Indent + Frm = '(1X, X,A)' + WRITE (Frm(5:7),'(I3)') Indent ELSE Frm = '(1X,A)' END IF