diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d6bfa66f..0cd03fa64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,6 +43,7 @@ option(USE_DLL_INTERFACE "Enable runtime loading of dynamic libraries" on) option(FPE_TRAP_ENABLED "Enable FPE trap in compiler options" off) option(WIN_DLL_LOAD "Enable loading of Windows only DLL's (OrcaFlex, SoilDyn)" on) # This is mostly for testing purposes option(BUILD_FASTFARM "Enable building FAST.Farm" off) +option(FASTFARM_TIMING_PRINTS "Enable FAST.Farm timing instrumentation and summary output" off) option(BUILD_OPENFAST_CPP_API "Enable building OpenFAST - C++ API" off) option(BUILD_OPENFAST_CPP_DRIVER "Enable building OpenFAST C++ driver using C++ CFD API" off) option(BUILD_OPENFAST_LIB_DRIVER "Enable building OpenFAST driver using C++ Library API" off) diff --git a/glue-codes/fast-farm/CMakeLists.txt b/glue-codes/fast-farm/CMakeLists.txt index 96672632b..c86bdbff5 100644 --- a/glue-codes/fast-farm/CMakeLists.txt +++ b/glue-codes/fast-farm/CMakeLists.txt @@ -29,6 +29,10 @@ add_executable(FAST.Farm src/FASTWrapper_Types.f90 ) target_link_libraries(FAST.Farm openfastlib_static wdlib awaelib) +if (FASTFARM_TIMING_PRINTS) + target_sources(FAST.Farm PRIVATE src/FAST_Farm_timings.f90) + target_compile_definitions(FAST.Farm PRIVATE FF_TIMING_PRINTS) +endif() set_property(TARGET FAST.Farm PROPERTY LINKER_LANGUAGE Fortran) string(TOUPPER ${CMAKE_Fortran_COMPILER_ID} _compiler_id) diff --git a/glue-codes/fast-farm/src/FAST_Farm.f90 b/glue-codes/fast-farm/src/FAST_Farm.f90 index c3d63ed75..4bf59344e 100644 --- a/glue-codes/fast-farm/src/FAST_Farm.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm.f90 @@ -53,6 +53,12 @@ PROGRAM FAST_Farm REAL(ReKi) :: SimStrtCPU !< User CPU time for simulation (without initialization) REAL(ReKi) :: ProgStrtCPU !< User CPU time for program (with initialization) +#ifdef FF_TIMING_PRINTS +REAL(DbKi) :: tmSer0 +REAL(DbKi) :: tmPar0 +REAL(DbKi) :: tmPar1 +#endif + ! these should probably go in the FAST.Farm registry: type(All_FastFarm_Data) :: farm @@ -74,6 +80,10 @@ PROGRAM FAST_Farm farm%p%NumTurbines = 0 t = 0 + +#ifdef FF_TIMING_PRINTS + call FarmTiming_Reset() +#endif InputFileName = "" ! make sure we don't think this is a "default" inputFileName if not specified on command line CALL CheckArgs( InputFileName, ErrStat=ErrStat, Flag=FlagArg ) @@ -85,7 +95,15 @@ PROGRAM FAST_Farm endif ! Initialize AMReX library +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call amrex_init(arg_parmparse=.false.) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('AMREX:Init', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif CALL FAST_ProgStart( Farm_Ver ) ! put this after CheckArgs because CheckArgs assumes we haven't called this routine, yet. @@ -101,8 +119,16 @@ PROGRAM FAST_Farm ! Initialization !............................................................................................................................... - call Farm_Initialize( farm, InputFileName, ErrStat, ErrMsg ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + call Farm_Initialize( farm, InputFileName, ErrStat, ErrMsg ) CALL CheckError( ErrStat, ErrMsg, 'during driver initialization' ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('INIT', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif !............................................................................................................................... ! Initial Calculate Output @@ -110,8 +136,16 @@ PROGRAM FAST_Farm CALL SimStatus_FirstTime( PrevSimTime, PrevClockTime, SimStrtTime, SimStrtCPU, t, farm%p%TMax ) - call FARM_InitialCO(farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + call FARM_InitialCO(farm, ErrStat, ErrMsg) CALL CheckError( ErrStat, ErrMsg, 'during initial calculate output' ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('ICO', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif END IF @@ -140,16 +174,32 @@ PROGRAM FAST_Farm t = n_t_global*farm%p%DT_low - CALL FARM_UpdateStates(t, n_t_global, farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + CALL FARM_UpdateStates(t, n_t_global, farm, ErrStat, ErrMsg) CALL CheckError( ErrStat, ErrMsg ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('US', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif t = (n_t_global+1)*farm%p%DT_low - CALL FARM_CalcOutput(t, farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + CALL FARM_CalcOutput(t, farm, ErrStat, ErrMsg) CALL CheckError( ErrStat, ErrMsg ) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('CO', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif CALL SimStatus( PrevSimTime, PrevClockTime, t, farm%p%TMax ) @@ -160,12 +210,32 @@ PROGRAM FAST_Farm ! End: !............................................................................................................................... - call FARM_End(farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + call FARM_End(farm, ErrStat, ErrMsg) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('END', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif ! Finalize AMReX library +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call amrex_finalize() +#ifdef FF_TIMING_PRINTS + call cpu_time(tmPar1) + call FarmTiming_Add('AMREX:Finalize', tmPar1-tmSer0, FarmTiming_WallTime()-tmPar0) +#endif CALL RunTimes( ProgStrtTime, ProgStrtCPU, SimStrtTime, SimStrtCPU, t ) + +#ifdef FF_TIMING_PRINTS + call FarmTiming_PrintSummary() +#endif call NormStop() CONTAINS diff --git a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 index 30a716047..4dbcc246b 100644 --- a/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 +++ b/glue-codes/fast-farm/src/FAST_Farm_Subs.f90 @@ -38,6 +38,9 @@ MODULE FAST_Farm_Subs #ifdef _OPENMP USE OMP_LIB #endif +#ifdef FF_TIMING_PRINTS + USE FAST_Farm_Timings +#endif IMPLICIT NONE @@ -153,10 +156,18 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) CHARACTER(*), PARAMETER :: RoutineName = 'Farm_Initialize' CHARACTER(ChanLen) :: OutList(Farm_MaxOutPts) ! list of user-requested output channels INTEGER(IntKi) :: i +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0, tmPar1, tmPar2 +#endif !.......... ErrStat = ErrID_None ErrMsg = "" AbortErrLev = ErrID_Fatal ! Until we read otherwise from the FAST input file, we abort only on FATAL errors + +#ifdef FF_TIMING_PRINTS + tmPar1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'Stage', 'Par') +#endif ! ... Open and read input files, initialize global parameters. ... @@ -242,9 +253,20 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) if (farm%p%WAT /= Mod_WAT_None .and. associated(farm%WAT_IfW%p%FlowField)) then AWAE_InitInput%WAT_FlowField => farm%WAT_IfW%p%FlowField endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'AWAE_Init', 'Par') + call AWAE_SetTimingStage('INIT') +#endif call AWAE_Init( AWAE_InitInput, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, farm%AWAE%OtherSt, farm%AWAE%y, & farm%AWAE%m, farm%p%DT_low, AWAE_InitOutput, ErrStat2, ErrMsg2 ) if(Failed()) return; +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:AWAE:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'AWAE_Init', 'Par', tmPar2-tmPar0) +#endif farm%AWAE%IsInitialized = .true. @@ -262,21 +284,51 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) !------------------- ! c. initialize WD (one instance per turbine, each can be done in parallel, too) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'WD_Init', 'Par') +#endif call Farm_InitWD( farm, WD_InitInput, ErrStat2, ErrMsg2 ); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:WD:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'WD_Init', 'Par', tmPar2-tmPar0) +#endif !............................................................................................................................... ! step 4: initialize FAST (each instance of FAST can also be done in parallel) !............................................................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'FWrap_Init', 'Par') +#endif CALL Farm_InitFAST( farm, WD_InitInput%InputFileData, AWAE_InitOutput, ErrStat2, ErrMsg2); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:FWrap:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'FWrap_Init', 'Par', tmPar2-tmPar0) +#endif !............................................................................................................................... ! step 4.5: initialize farm-level MoorDyn if applicable !............................................................................................................................... if (farm%p%MooringMod == 3) then +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('INIT', 'MD_Init', 'Serial') +#endif CALL Farm_InitMD( farm, ErrStat2, ErrMsg2); if(Failed()) return; ! FAST instances must be initialized first so that turbine initial positions are known +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('INIT:MD:Init', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('INIT', 'MD_Init', 'Serial', tmPar2-tmPar0) +#endif end if !............................................................................................................................... @@ -284,19 +336,45 @@ SUBROUTINE Farm_Initialize( farm, InputFile, ErrStat, ErrMsg ) !............................................................................................................................... ! Set parameters for output channels: +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif CALL Farm_SetOutParam(OutList, farm, ErrStat2, ErrMsg2 ); if(Failed()) return; ! requires: p%NumOuts, sets: p%OutParam. +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('INIT:IO:SetOutParam', tmSer0, tmPar0) +#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_InitOutput( farm, ErrStat2, ErrMsg2 ); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('INIT:IO:InitOutput', tmSer0, tmPar0) +#endif ! Print the summary file if requested: IF (farm%p%SumPrint) THEN +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif CALL Farm_PrintSum( farm, WD_InitInput%InputFileData, ErrStat2, ErrMsg2 ); if(Failed()) return; +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('INIT:IO:PrintSummary', tmSer0, tmPar0) +#endif END IF !............................................................................................................................... ! Destroy initializion data !............................................................................................................................... CALL Cleanup() + +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('INIT', 'Stage', 'Par', tmPar2-tmPar1) +#endif CONTAINS SUBROUTINE Cleanup() @@ -1042,10 +1120,18 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) INTEGER(IntKi) :: ErrStat2 ! Temporary Error status CHARACTER(ErrMsgLen) :: ErrMsg2 ! Temporary Error message CHARACTER(*), PARAMETER :: RoutineName = 'FARM_InitialCO' +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0, tmPar1, tmPar2 +#endif ErrStat = ErrID_None ErrMsg = "" + +#ifdef FF_TIMING_PRINTS + tmPar1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'Stage', 'Par') +#endif @@ -1065,10 +1151,22 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) !-------------------- ! 1b. CALL AWAE_CO +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'AWAE_CalcOutput1', 'Par') + call AWAE_SetTimingStage('ICO:AWAE:CalcOutput1') +#endif call AWAE_CalcOutput( 0.0_DbKi, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('ICO:AWAE:CalcOutput1', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'AWAE_CalcOutput1', 'Par', tmPar2-tmPar0) +#endif !-------------------- ! 1c. transfer y_AWAE to u_F and u_WD @@ -1078,6 +1176,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) ! CALL F_t0 (can be done in parallel) !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'FWrap_t0', 'Par') +#endif DO nt = 1,farm%p%NumTurbines call FWrap_t0( farm%FWrap(nt)%u, farm%FWrap(nt)%p, farm%FWrap(nt)%x, farm%FWrap(nt)%xd, farm%FWrap(nt)%z, & @@ -1086,6 +1189,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) END DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('ICO:FWrap:t0', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'FWrap_t0', 'Par', tmPar2-tmPar0) +#endif !....................................................................................... ! Transfer y_F to u_WD @@ -1100,6 +1208,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) ! CALL WD_CO (can be done in parallel) !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'WD_CalcOutput', 'Par') +#endif DO nt = 1,farm%p%NumTurbines call WD_CalcOutput( 0.0_DbKi, farm%WD(nt)%u, farm%WD(nt)%p, farm%WD(nt)%x, farm%WD(nt)%xd, farm%WD(nt)%z, & @@ -1108,6 +1221,11 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) END DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('ICO:WD:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'WD_CalcOutput', 'Par', tmPar2-tmPar0) +#endif !....................................................................................... ! Transfer y_WD to u_AWAE @@ -1118,11 +1236,22 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) !....................................................................................... ! CALL AWAE_CO !....................................................................................... - +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('ICO', 'AWAE_CalcOutput2', 'Par') + call AWAE_SetTimingStage('ICO:AWAE:CalcOutput2') +#endif call AWAE_CalcOutput( 0.0_DbKi, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tmPar2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('ICO:AWAE:CalcOutput2', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('ICO', 'AWAE_CalcOutput2', 'Par', tmPar2-tmPar0) +#endif !....................................................................................... ! Transfer y_AWAE to u_F and u_WD @@ -1134,8 +1263,18 @@ subroutine FARM_InitialCO(farm, ErrStat, ErrMsg) ! Write Output to File !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_WriteOutput(0, 0.0_DbKi, farm, ErrStat2, ErrMsg2) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) + +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('ICO:IO:WriteOutput', tmSer0, tmPar0) + tmPar2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('ICO', 'Stage', 'Par', tmPar2-tmPar1) +#endif end subroutine FARM_InitialCO !---------------------------------------------------------------------------------------------------------------------------------- @@ -1158,14 +1297,19 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) INTEGER(IntKi) :: n_ss INTEGER(IntKi) :: n_FMD REAL(DbKi) :: t2 ! time within the FAST-MoorDyn substepping loop for shared moorings - INTEGER(IntKi) :: ErrStatMD, ErrStat2 + INTEGER(IntKi) :: ErrStatMD, ErrStatAWAE, ErrStat2 CHARACTER(ErrMsgLen) :: ErrMsg2 CHARACTER(ErrMsgLen) :: ErrMsgAWAE CHARACTER(ErrMsgLen) :: ErrMsgMD INTEGER(IntKi), ALLOCATABLE :: ErrStatF(:) ! Temporary Error status for FAST CHARACTER(ErrMsgLen), ALLOCATABLE :: ErrMsgF (:) ! Temporary Error message for FAST CHARACTER(*), PARAMETER :: RoutineName = 'FARM_UpdateStates' - REAL(DbKi) :: tm1,tm2,tm3, tm01, tm02, tm03, tmSF, tmSM ! timer variables + REAL(DbKi) :: tm1, tm2 +#ifdef FF_TIMING_PRINTS + REAL(DbKi) :: tmSer0, tmPar0, tmPar1 + REAL(DbKi), ALLOCATABLE :: fwrapThreadDt(:) + INTEGER(IntKi), ALLOCATABLE :: fwrapThreadId(:) +#endif ErrStat = ErrID_None ErrMsg = "" @@ -1175,6 +1319,11 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) allocate ( ErrMsgF ( farm%p%NumTurbines ), STAT=errStat2 ) if (errStat2 /= 0) call SetErrStat ( ErrID_Fatal, 'Could not allocate memory for ErrMsgF.', errStat, errMsg, RoutineName ) if (ErrStat >= AbortErrLev) return + +#ifdef FF_TIMING_PRINTS + tmPar1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'Stage', 'Par') +#endif @@ -1186,7 +1335,13 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) !-------------------- ! 1. CALL WD_US - + +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'WD_UpdateStates', 'Par') +#endif + !$OMP PARALLEL default(shared) !$OMP do private(nt, ErrStat2, ErrMsg2) schedule(runtime) DO nt = 1,farm%p%NumTurbines @@ -1205,6 +1360,12 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) END DO !$OMP END DO !$OMP END PARALLEL + +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('US:WD:UpdateStates', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'WD_UpdateStates', 'Par', tm2-tmPar0) +#endif if (ErrStat >= AbortErrLev) return @@ -1213,9 +1374,16 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! set the inputs needed for FAST (these are slow-varying so can just be done once per farm time step) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif do nt = 1,farm%p%NumTurbines call FWrap_SetWindTStart(farm%FWrap(nt)%u, farm%FWrap(nt)%m, t) end do +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('US:FWrap:SetWindTStart', tmSer0, tmPar0) +#endif !#ifdef printthreads @@ -1226,12 +1394,52 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! Original case: no shared moorings if (farm%p%MooringMod == 0) then +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'FWrap_Increment_NoMD', 'Par') + + allocate(fwrapThreadDt(farm%p%NumTurbines), fwrapThreadId(farm%p%NumTurbines), stat=ErrStat2) + if (ErrStat2 /= 0) then + call SetErrStat(ErrID_Fatal, 'Could not allocate memory for buffered FWrap thread timing output.', ErrStat, ErrMsg, RoutineName) + return + end if + fwrapThreadDt = -1.0_DbKi + fwrapThreadId = -1_IntKi +#endif + !$OMP PARALLEL DO DEFAULT(Shared) Private(nt) DO nt = 1,farm%p%NumTurbines +#ifdef FF_TIMING_PRINTS + fwrapThreadDt(nt) = FarmTiming_WallTime() +#endif call FWrap_Increment( t, n, farm%FWrap(nt)%u, farm%FWrap(nt)%p, farm%FWrap(nt)%x, farm%FWrap(nt)%xd, farm%FWrap(nt)%z, & farm%FWrap(nt)%OtherSt, farm%FWrap(nt)%y, farm%FWrap(nt)%m, ErrStatF(nt), ErrMsgF(nt) ) +#ifdef FF_TIMING_PRINTS + fwrapThreadDt(nt) = FarmTiming_WallTime() - fwrapThreadDt(nt) +#ifdef _OPENMP + fwrapThreadId(nt) = omp_get_thread_num() +#else + fwrapThreadId(nt) = 0_IntKi +#endif +#endif END DO !$OMP END PARALLEL DO + +#ifdef FF_TIMING_PRINTS + do nt = 1, farm%p%NumTurbines + if (fwrapThreadDt(nt) >= 0.0_DbKi) then + call FarmTiming_PrintLiveThread('US', 'FWrap_Increment_NoMD', nt, fwrapThreadId(nt), fwrapThreadDt(nt)) + end if + end do + deallocate(fwrapThreadDt, fwrapThreadId) +#endif + +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('US:FWrap:Increment', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'FWrap_Increment_NoMD', 'Par', tm2-tmPar0) +#endif ! Farm-level moorings case using MoorDyn else if (farm%p%MooringMod == 3) then @@ -1247,6 +1455,10 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! tm01 = omp_get_wtime() !#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif ! A nested parallel for loop to call each instance of OpenFAST in parallel !$OMP PARALLEL DO DEFAULT(Shared) Private(nt) DO nt = 1,farm%p%NumTurbines @@ -1254,14 +1466,26 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) farm%FWrap(nt)%OtherSt, farm%FWrap(nt)%y, farm%FWrap(nt)%m, ErrStatF(nt), ErrMsgF(nt) ) END DO !$OMP END PARALLEL DO + +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('US:FWrap:Increment', tmSer0, tmPar0) +#endif !#ifdef printthreads ! tm02 = omp_get_wtime() !#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif ! call farm-level MoorDyn time step here (can't multithread this with FAST since it needs inputs from all FAST instances) call Farm_MD_Increment( t2, n_FMD, farm, ErrStatMD, ErrMsgMD) call SetErrStat(ErrStatMD, ErrMsgMD, ErrStat, ErrMsg, 'FARM_UpdateStates') ! MD error status <<<<< + +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('US:MD:Increment', tmSer0, tmPar0) +#endif !#ifdef printthreads ! tm03 = omp_get_wtime() @@ -1285,6 +1509,28 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) ! write(*,*) 'Total FAST and Moordyn for FF_US took '//trim(num2lstr(tm2-tm1))//' seconds.' !#endif +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'AWAE_UpdateStates', 'Par') + call AWAE_SetTimingStage('US:AWAE:UpdateStates') +#endif + call AWAE_UpdateStates( n, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & + farm%AWAE%OtherSt, farm%AWAE%m, ErrStatAWAE, ErrMsgAWAE ) + call SetErrStat(ErrStatAWAE, ErrMsgAWAE, ErrStat, ErrMsg, 'FARM_UpdateStates') ! AWAE error status + +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('US:AWAE:UpdateStates', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'AWAE_UpdateStates', 'Par', tm2-tmPar0) +#endif + + !#ifdef printthreads + ! tm3 = omp_get_wtime() + ! write(*,*) 'AWAE_US took '//trim(num2lstr(tm3-tm2))//' seconds.' + ! write(*,*) 'Total Farm_US took '//trim(num2lstr(tm3-tm1))//' seconds.' + !#endif ! update error messages from FAST's and AWAE's time steps DO nt = 1,farm%p%NumTurbines @@ -1292,14 +1538,30 @@ subroutine FARM_UpdateStates(t, n, farm, ErrStat, ErrMsg) END DO ! calculate outputs from FAST as needed by FAST.Farm +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('US', 'FWrap_CalcOutput', 'Serial') +#endif do nt = 1,farm%p%NumTurbines call FWrap_CalcOutput(farm%FWrap(nt)%p, farm%FWrap(nt)%u, farm%FWrap(nt)%y, farm%FWrap(nt)%m, ErrStat2, ErrMsg2) call setErrStat(ErrStat2,ErrMsg2,ErrStat,ErrMsg,RoutineName) end do +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('US:FWrap:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('US', 'FWrap_CalcOutput', 'Serial', tm2-tmPar0) +#endif + if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_PrintLiveDone('US', 'Stage', 'Par', tm2-tmPar1) +#endif + end subroutine FARM_UpdateStates !---------------------------------------------------------------------------------------------------------------------------------- @@ -1587,7 +1849,10 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) CHARACTER(ErrMsgLen) :: ErrMsg2 ! Temporary Error message CHARACTER(*), PARAMETER :: RoutineName = 'FARM_CalcOutput' INTEGER(IntKi) :: n ! time step increment number -! REAL(DbKi) :: tm1 + REAL(DbKi) :: tm1, tm2 +#ifdef FF_TIMING_PRINTS + REAL(DbKi) :: tmSer0, tmPar0 +#endif ErrStat = ErrID_None ErrMsg = "" @@ -1602,7 +1867,15 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !....................................................................................... !-------------------- - ! 1. call WD_CO and transfer y_WD to u_AWAE + ! 1. call WD_CO and transfer y_WD to u_AWAE + +#ifdef FF_TIMING_PRINTS + tm1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'Stage', 'Serial') + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'WD_CalcOutput', 'Par') +#endif !$OMP PARALLEL DO DEFAULT (shared) PRIVATE(nt, ErrStat2, ErrMsg2) schedule(runtime) DO nt = 1,farm%p%NumTurbines @@ -1618,7 +1891,17 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !$OMP END PARALLEL DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('CO:WD:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'WD_CalcOutput', 'Par', tm2-tmPar0) +#endif + ! IO operation, not done using OpenMP +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif DO nt = 1,farm%p%NumTurbines call WD_WritePlaneOutputs( t, farm%WD(nt)%u, farm%WD(nt)%p, farm%WD(nt)%x, farm%WD(nt)%xd, farm%WD(nt)%z, & farm%WD(nt)%OtherSt, farm%WD(nt)%y, farm%WD(nt)%m, ErrStat2, ErrMsg2 ) @@ -1628,13 +1911,31 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) END DO if (ErrStat >= AbortErrLev) return +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:WD:WritePlaneOutputs', tmSer0, tmPar0) +#endif + +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Transfer_WD_to_AWAE(farm) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:Transfer:WD_to_AWAE', tmSer0, tmPar0) +#endif !-------------------- ! 2. Transfer y_F to u_WD - + +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Transfer_FAST_to_WD(farm) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:Transfer:FAST_to_WD', tmSer0, tmPar0) +#endif !....................................................................................... ! calculate AWAE outputs and perform rest of input-output solves @@ -1642,31 +1943,77 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) !-------------------- ! 0. call AWAE_UpdateStates to get the ambient wind and calculate wake-grid interactions +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'AWAE_UpdateStates', 'Par') + call AWAE_SetTimingStage('CO:AWAE:UpdateStates') +#endif call AWAE_UpdateStates( n, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('CO:AWAE:UpdateStates', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'AWAE_UpdateStates', 'Par', tm2-tmPar0) +#endif !-------------------- ! 1. call AWAE_CO +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('CO', 'AWAE_CalcOutput', 'Par') + call AWAE_SetTimingStage('CO:AWAE:CalcOutput') +#endif call AWAE_CalcOutput( t, farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('CO:AWAE:CalcOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'AWAE_CalcOutput', 'Par', tm2-tmPar0) +#endif + !-------------------- ! 2. Transfer y_AWAE to u_F and u_WD +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Transfer_AWAE_to_WD(farm) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('CO:Transfer:AWAE_to_WD', tmSer0, tmPar0) +#endif !....................................................................................... ! Write Output to File !....................................................................................... ! NOTE: Visualization data is output via the AWAE module +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif + n = nint(t/farm%p%DT_low) call Farm_WriteOutput(n, t, farm, ErrStat2, ErrMsg2) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('CO:IO:WriteOutput', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'WriteOutput', 'Serial', tm2-tmPar0) +#endif !....................................................................................... ! Write shared moorings visualization !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif ! Write visualization meshes if (farm%p%MooringMod == 3) then @@ -1689,6 +2036,13 @@ subroutine FARM_CalcOutput(t, farm, ErrStat, ErrMsg) endif endif +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('CO:IO:WriteMooringVis', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('CO', 'WriteMooringVis', 'Serial', tm2-tmPar0) + call FarmTiming_PrintLiveDone('CO', 'Stage', 'Serial', tm2-tm1) +#endif + ! write(*,*) 'Total Farm_CO-serial took '//trim(num2lstr(omp_get_wtime()-tm1))//' seconds.' end subroutine FARM_CalcOutput @@ -1710,11 +2064,18 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) INTEGER(IntKi) :: ErrStat2 ! Temporary Error status CHARACTER(ErrMsgLen) :: ErrMsg2 ! Temporary Error message CHARACTER(*), PARAMETER :: RoutineName = 'FARM_End' - - +#ifdef FF_TIMING_PRINTS + REAL(DbKi) :: tm1, tm2 + REAL(DbKi) :: tmSer0, tmPar0 +#endif ErrStat = ErrID_None ErrMsg = "" + +#ifdef FF_TIMING_PRINTS + tm1 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('END', 'Stage', 'Serial') +#endif !....................................................................................... ! end all modules (1-4 can be done in parallel) @@ -1722,24 +2083,47 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) !-------------- ! 1. end AWAE +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (farm%WAT_IfW%IsInitialized) then call InflowWind_End(farm%WAT_IfW%u, farm%WAT_IfW%p, farm%WAT_IfW%x, farm%WAT_IfW%xd, farm%WAT_IfW%z, & farm%WAT_IfW%OtherSt, farm%WAT_IfW%y, farm%WAT_IfW%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) farm%WAT_IfW%IsInitialized = .false. endif +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:WAT:End', tmSer0, tmPar0) +#endif !-------------- ! 2. end AWAE +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() + call FarmTiming_PrintLiveStart('END', 'AWAE_End', 'Par') + call AWAE_SetTimingStage('END') +#endif if (farm%AWAE%IsInitialized) then call AWAE_End( farm%AWAE%u, farm%AWAE%p, farm%AWAE%x, farm%AWAE%xd, farm%AWAE%z, & farm%AWAE%OtherSt, farm%AWAE%y, farm%AWAE%m, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) farm%AWAE%IsInitialized = .false. end if +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddElapsed('END:AWAE:End', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('END', 'AWAE_End', 'Par', tm2-tmPar0) +#endif !-------------- ! 3. end WakeDynamics +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (allocated(farm%WD)) then DO nt = 1,farm%p%NumTurbines if (farm%WD(nt)%IsInitialized) then @@ -1750,9 +2134,16 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) end if END DO end if +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:WD:End', tmSer0, tmPar0) +#endif !-------------- ! 5. End each instance of FAST (each instance of FAST can be done in parallel, too) +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (allocated(farm%FWrap)) then DO nt = 1,farm%p%NumTurbines if (farm%FWrap(nt)%IsInitialized) then @@ -1763,28 +2154,54 @@ subroutine FARM_End(farm, ErrStat, ErrMsg) end if END DO end if +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:FWrap:End', tmSer0, tmPar0) +#endif !-------------- ! 6. End farm-level MoorDyn +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif if (farm%p%MooringMod == 3 .and. allocated(farm%MD%Input)) then call MD_End(farm%MD%Input(1), farm%MD%p, farm%MD%x, farm%MD%xd, farm%MD%z, farm%MD%OtherSt, farm%MD%y, farm%MD%m, ErrStat2, ErrMsg2) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName ) !TODO: any related items need to be cleared? end if +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:MD:End', tmSer0, tmPar0) +#endif !....................................................................................... ! close output file !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_EndOutput( farm, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg,RoutineName) +#ifdef FF_TIMING_PRINTS + call FarmTiming_AddElapsed('END:IO:EndOutput', tmSer0, tmPar0) +#endif !....................................................................................... ! clear all data from 'farm' structure !....................................................................................... +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = FarmTiming_WallTime() +#endif call Farm_DestroyAll_FastFarm_Data( farm, ErrStat2, ErrMsg2 ) call SetErrStat(ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName) +#ifdef FF_TIMING_PRINTS + tm2 = FarmTiming_WallTime() + call FarmTiming_AddElapsed('END:Farm:DestroyData', tmSer0, tmPar0) + call FarmTiming_PrintLiveDone('END', 'Stage', 'Serial', tm2-tm1) +#endif end subroutine FARM_End !---------------------------------------------------------------------------------------------------------------------------------- diff --git a/glue-codes/fast-farm/src/FAST_Farm_timings.f90 b/glue-codes/fast-farm/src/FAST_Farm_timings.f90 new file mode 100644 index 000000000..95e859325 --- /dev/null +++ b/glue-codes/fast-farm/src/FAST_Farm_timings.f90 @@ -0,0 +1,273 @@ +!********************************************************************************************************************************** +! LICENSING +! Copyright (C) 2026 National Renewable Energy Laboratory +! +! This file is part of FAST.Farm. +! +! Licensed under the Apache License, Version 2.0 (the "License"); +! you may not use this file except in compliance with the License. +! You may obtain a copy of the License at +! +! http://www.apache.org/licenses/LICENSE-2.0 +! +! Unless required by applicable law or agreed to in writing, software +! distributed under the License is distributed on an "AS IS" BASIS, +! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +! See the License for the specific language governing permissions and +! limitations under the License. +! +!********************************************************************************************************************************** +MODULE FAST_Farm_Timings + + USE NWTC_Library + USE AWAE_Timings, only: AWAE_ResetTiming, AWAE_DrainTimingData, AWAE_MaxTimingEntries + +#ifdef _OPENMP + USE OMP_LIB +#endif + + IMPLICIT NONE + + integer(IntKi), parameter :: FFTiming_MaxEntries = 256 + integer(IntKi), save :: FFTiming_Count = 0 + character(128), save :: FFTiming_Label(FFTiming_MaxEntries) + real(DbKi), save :: FFTiming_Serial(FFTiming_MaxEntries) = 0.0_DbKi + real(DbKi), save :: FFTiming_Par(FFTiming_MaxEntries) = 0.0_DbKi + +CONTAINS + +subroutine FarmTiming_Reset() + call AWAE_ResetTiming() + FFTiming_Count = 0 + FFTiming_Label = '' + FFTiming_Serial = 0.0_DbKi + FFTiming_Par = 0.0_DbKi +end subroutine FarmTiming_Reset + +real(DbKi) function FarmTiming_WallTime() +#ifdef _OPENMP + FarmTiming_WallTime = omp_get_wtime() +#else + call cpu_time(FarmTiming_WallTime) +#endif +end function FarmTiming_WallTime + +subroutine FarmTiming_Add(label, serialDt, parDt) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialDt + real(DbKi), intent(in) :: parDt + integer(IntKi) :: i + + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(label)) then + FFTiming_Serial(i) = FFTiming_Serial(i) + serialDt + FFTiming_Par(i) = FFTiming_Par(i) + parDt + return + end if + end do + + if (FFTiming_Count < FFTiming_MaxEntries) then + FFTiming_Count = FFTiming_Count + 1 + FFTiming_Label(FFTiming_Count) = label + FFTiming_Serial(FFTiming_Count) = serialDt + FFTiming_Par(FFTiming_Count) = parDt + end if +end subroutine FarmTiming_Add + +subroutine FarmTiming_AddElapsed(label, serialStart, parStart) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialStart + real(DbKi), intent(in) :: parStart + real(DbKi) :: serialEnd, parEnd + + call cpu_time(serialEnd) + parEnd = FarmTiming_WallTime() + call FarmTiming_Add(label, serialEnd-serialStart, parEnd-parStart) +end subroutine FarmTiming_AddElapsed + +subroutine FarmTiming_DrainAWAEDetails() + integer(IntKi) :: n, i + character(128) :: labels(AWAE_MaxTimingEntries) + real(DbKi) :: serial(AWAE_MaxTimingEntries) + real(DbKi) :: par(AWAE_MaxTimingEntries) + + call AWAE_DrainTimingData(n, labels, serial, par) + do i = 1, n + call FarmTiming_Add(trim(labels(i)), serial(i), par(i)) + end do +end subroutine FarmTiming_DrainAWAEDetails + +logical function FarmTiming_IsStageChild(label, stage) + character(*), intent(in) :: label + character(*), intent(in) :: stage + character(24) :: prefix + integer(IntKi) :: nColons + + prefix = trim(stage)//':' + if (index(trim(label), trim(prefix)) /= 1) then + FarmTiming_IsStageChild = .false. + return + end if + + nColons = FarmTiming_CountColons(label) + FarmTiming_IsStageChild = (nColons == 2) +end function FarmTiming_IsStageChild + +integer(IntKi) function FarmTiming_CountColons(label) + character(*), intent(in) :: label + integer(IntKi) :: i + + FarmTiming_CountColons = 0 + do i = 1, len_trim(label) + if (label(i:i) == ':') FarmTiming_CountColons = FarmTiming_CountColons + 1 + end do +end function FarmTiming_CountColons + +logical function FarmTiming_IsParentChild(label, parent) + character(*), intent(in) :: label + character(*), intent(in) :: parent + character(128) :: prefix + integer(IntKi) :: parentColons, labelColons + + prefix = trim(parent)//':' + if (index(trim(label), trim(prefix)) /= 1) then + FarmTiming_IsParentChild = .false. + return + end if + + parentColons = FarmTiming_CountColons(parent) + labelColons = FarmTiming_CountColons(label) + FarmTiming_IsParentChild = (labelColons == parentColons + 1) +end function FarmTiming_IsParentChild + +subroutine FarmTiming_AddParentOthers(parent) + character(*), intent(in) :: parent + integer(IntKi) :: i, iParent + real(DbKi) :: serialChildren, parChildren + real(DbKi) :: serialOthers, parOthers + character(128) :: othersLabel + + iParent = 0 + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(parent)) then + iParent = i + exit + end if + end do + if (iParent == 0) return + + serialChildren = 0.0_DbKi + parChildren = 0.0_DbKi + do i = 1, FFTiming_Count + if (FarmTiming_IsParentChild(FFTiming_Label(i), parent)) then + serialChildren = serialChildren + FFTiming_Serial(i) + parChildren = parChildren + FFTiming_Par(i) + end if + end do + + serialOthers = FFTiming_Serial(iParent) - serialChildren + parOthers = FFTiming_Par(iParent) - parChildren + if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi + if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi + + othersLabel = trim(parent)//':Others' + call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) +end subroutine FarmTiming_AddParentOthers + +subroutine FarmTiming_AddStageOthers(stage) + character(*), intent(in) :: stage + integer(IntKi) :: i + integer(IntKi) :: iStage + real(DbKi) :: serialChildren, parChildren + real(DbKi) :: serialOthers, parOthers + character(32) :: othersLabel + + iStage = 0 + do i = 1, FFTiming_Count + if (trim(FFTiming_Label(i)) == trim(stage)) then + iStage = i + exit + end if + end do + if (iStage == 0) return + + serialChildren = 0.0_DbKi + parChildren = 0.0_DbKi + do i = 1, FFTiming_Count + if (FarmTiming_IsStageChild(FFTiming_Label(i), stage)) then + serialChildren = serialChildren + FFTiming_Serial(i) + parChildren = parChildren + FFTiming_Par(i) + end if + end do + + serialOthers = FFTiming_Serial(iStage) - serialChildren + parOthers = FFTiming_Par(iStage) - parChildren + if (abs(serialOthers) < 1.0e-9_DbKi) serialOthers = 0.0_DbKi + if (abs(parOthers) < 1.0e-9_DbKi) parOthers = 0.0_DbKi + + othersLabel = trim(stage)//':Others' + call FarmTiming_Add(trim(othersLabel), serialOthers, parOthers) +end subroutine FarmTiming_AddStageOthers + +subroutine FarmTiming_PrintLiveStart(stage, label, mode) + character(*), intent(in) :: stage + character(*), intent(in) :: label + character(*), intent(in) :: mode + character(384) :: line + + line = 'FFTiming|LIVE|Event=Start|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveStart + +subroutine FarmTiming_PrintLiveDone(stage, label, mode, dt) + character(*), intent(in) :: stage + character(*), intent(in) :: label + character(*), intent(in) :: mode + real(DbKi), intent(in) :: dt + character(384) :: line + + line = 'FFTiming|LIVE|Event=Done|Stage='//trim(stage)//'|Label='//trim(label)//'|Mode='//trim(mode)//'|Seconds='//trim(num2lstr(dt)) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveDone + +subroutine FarmTiming_PrintLiveThread(stage, label, itemId, threadId, dt) + character(*), intent(in) :: stage + character(*), intent(in) :: label + integer(IntKi), intent(in) :: itemId + integer(IntKi), intent(in) :: threadId + real(DbKi), intent(in) :: dt + character(384) :: line + + line = 'FFTiming|LIVE|Event=ThreadDone|Stage='//trim(stage)//'|Label='//trim(label)//'|Item='//trim(num2lstr(itemId))// & + '|Thread='//trim(num2lstr(threadId))//'|Seconds='//trim(num2lstr(dt)) + call WrScr(trim(line)) +end subroutine FarmTiming_PrintLiveThread + +subroutine FarmTiming_PrintSummary() + integer(IntKi) :: i + character(384) :: line + + call FarmTiming_DrainAWAEDetails() + call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput1') + call FarmTiming_AddParentOthers('ICO:AWAE:CalcOutput2') + call FarmTiming_AddParentOthers('US:AWAE:UpdateStates') + call FarmTiming_AddParentOthers('CO:AWAE:UpdateStates') + call FarmTiming_AddParentOthers('CO:AWAE:CalcOutput') + call FarmTiming_AddStageOthers('INIT') + call FarmTiming_AddStageOthers('ICO') + call FarmTiming_AddStageOthers('US') + call FarmTiming_AddStageOthers('CO') + call FarmTiming_AddStageOthers('END') + + call WrScr('') + call WrScr('FFTiming|SUMMARY|Begin') + call WrScr('FFTiming|SUMMARY|Columns=Label|CPU|Wall') + do i = 1, FFTiming_Count + line = 'FFTiming|SUMMARY|Label='//trim(FFTiming_Label(i))//'|CPU='//trim(num2lstr(FFTiming_Serial(i)))//'|Wall='//trim(num2lstr(FFTiming_Par(i))) + call WrScr(trim(line)) + end do + call WrScr('FFTiming|SUMMARY|End') + call WrScr('') +end subroutine FarmTiming_PrintSummary + +END MODULE FAST_Farm_Timings diff --git a/modules/awae/CMakeLists.txt b/modules/awae/CMakeLists.txt index fdb66b91b..2079ef67a 100644 --- a/modules/awae/CMakeLists.txt +++ b/modules/awae/CMakeLists.txt @@ -37,6 +37,11 @@ if (AMREX_READER) target_compile_definitions(awaelib PRIVATE ENABLE_AMREX_LIB) endif() +if (FASTFARM_TIMING_PRINTS) + target_sources(awaelib PRIVATE src/AWAE_timings.f90) + target_compile_definitions(awaelib PRIVATE FF_TIMING_PRINTS) +endif() + install(TARGETS awaelib awaelib_c EXPORT "${CMAKE_PROJECT_NAME}Libraries" RUNTIME DESTINATION bin diff --git a/modules/awae/src/AWAE.f90 b/modules/awae/src/AWAE.f90 index 0f8360cc5..8b329cd77 100644 --- a/modules/awae/src/AWAE.f90 +++ b/modules/awae/src/AWAE.f90 @@ -34,6 +34,9 @@ module AWAE #ifdef _OPENMP use OMP_LIB #endif +#ifdef FF_TIMING_PRINTS + use AWAE_Timings +#endif implicit none @@ -48,6 +51,17 @@ module AWAE ! continuous states, and updating discrete states public :: AWAE_CalcOutput ! Routine for computing outputs public :: AWAE_CalcConstrStateResidual ! Tight coupling routine for returning the constraint state residual +#ifdef FF_TIMING_PRINTS + ! exposing the AWAE_timings module public subroutines for use in FAST.Farm + public :: AWAE_ResetTiming ! Reset compile-time timing accumulators + public :: AWAE_GetTimingData ! Retrieve compile-time timing accumulators + public :: AWAE_DrainTimingData ! Retrieve and reset compile-time timing accumulators + public :: AWAE_SetTimingStage ! Set timing label prefix used by AWAE_CalcOutput timing + public :: AWAE_MaxTimingEntries + public :: AWAE_WallTime + public :: AWAE_AddTiming + public :: AWAE_AddStageTiming +#endif ! Unit testing routines @@ -1724,10 +1738,16 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) real(ReKi), allocatable :: AccUVW(:,:) logical :: WriteWindVTK real(DbKi) :: t - +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0 +#endif + errStat = ErrID_None errMsg = "" - + +#ifdef FF_TIMING_PRINTS + if (len_trim(AWAE_TimingPrefix) == 0) AWAE_TimingPrefix = 'US:AWAE:UpdateStates' +#endif ! If last time step, don't populate high-resolution grid if (n == (p%NumDT - 1)) then n_high_low = 0 @@ -1742,6 +1762,11 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) ! Populate low resolution grids based on ambient wind source !---------------------------------------------------------------------------- +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + select case (p%Mod_AmbWind) ! File-based ambient wind @@ -1772,10 +1797,19 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) end select +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('PopulateLowRes', tmSer0, tmPar0) +#endif + !---------------------------------------------------------------------------- ! Populate high-resolution grid based on ambient wind source !---------------------------------------------------------------------------- +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + select case (p%Mod_AmbWind) ! File-based ambient wind @@ -1897,12 +1931,21 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) end select +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('PopulateHighRes', tmSer0, tmPar0) +#endif + !---------------------------------------------------------------------------- ! Propagate WAT tracer !---------------------------------------------------------------------------- if (p%WAT_Enabled) then +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + ! Find mean velocity of all turbine disks xd%Ufarm = 0.0_ReKi do nt=1,p%NumTurbines @@ -1912,6 +1955,10 @@ subroutine AWAE_UpdateStates(n, u, p, x, xd, z, OtherState, m, errStat, errMsg) ! add mean velocity * dt to the tracer for the position of the WAT box xd%WAT_B_Box = xd%WAT_B_Box + xd%Ufarm*real(p%dt_low,ReKi) + +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('PropagateWAT', tmSer0, tmPar0) +#endif endif contains @@ -1957,6 +2004,9 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg CHARACTER(1024) :: FileName INTEGER(IntKi) :: Un ! unit number of opened file logical :: WriteWindVTK +#ifdef FF_TIMING_PRINTS + real(DbKi) :: tmSer0, tmPar0 +#endif errStat = ErrID_None errMsg = "" @@ -1964,7 +2014,14 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg ! some variables and indexing n = nint(t / p%dt_low) n_high = n*p%n_high_low - call ComputeLocals(n, u, p, y, m, ErrStat2, ErrMsg2); if (Failed()) return; +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + call ComputeLocals(n, u, p, y, m, errStat2, errMsg2); if (Failed()) return; +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('ComputeLocals', tmSer0, tmPar0) +#endif ! Set flag to write wind VTK files if it's the correct step WriteWindVTK = mod(n, p%WrDisSkp1) == 0 @@ -1975,18 +2032,42 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg ! m%iPlaneTurbTurb(2,p%NumTurbines,p%NumTurbines) (High-res grid) !---------------------------------------------------------------------------- +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif call CalcWakePointTurbineGridInteractions(p, m, u) +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('CalcWakePointTurbineGridInteractions', tmSer0, tmPar0) +#endif - ! High-resolution grid output - call HighResGridCalcOutput(n_high, u, p, xd, y, m, ErrStat2, ErrMsg2) - if (Failed()) return + ! high-res +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif + call HighResGridCalcOutput(n_high, u, p, xd, y, m, errStat2, errMsg2); if (Failed()) return; +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('HighResGridCalcOutput', tmSer0, tmPar0) +#endif ! Low-resolution grid output +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif call LowResGridCalcOutput(n, u, p, xd, y, m, ErrStat2, ErrMsg2) if (Failed()) return +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('LowResGridCalcOutput', tmSer0, tmPar0) +#endif ! If it's time to write wind VTK files if (WriteWindVTK) then +#ifdef FF_TIMING_PRINTS + call cpu_time(tmSer0) + tmPar0 = AWAE_WallTime() +#endif if (p%WrDisWind) then call WriteDisWindFiles(n, p%WrDisSkp1, p, y, m, ErrStat2, ErrMsg2) @@ -2046,6 +2127,10 @@ subroutine AWAE_CalcOutput( t, u, p, x, xd, z, OtherState, y, m, errStat, errMsg p%LowRes%dXYZ, m%outVizXZPlane, ErrStat2, ErrMsg2) if (Failed()) return end do + +#ifdef FF_TIMING_PRINTS + call AWAE_AddStageTiming('WriteDisWind', tmSer0, tmPar0) +#endif end if contains diff --git a/modules/awae/src/AWAE_timings.f90 b/modules/awae/src/AWAE_timings.f90 new file mode 100644 index 000000000..f654d5330 --- /dev/null +++ b/modules/awae/src/AWAE_timings.f90 @@ -0,0 +1,136 @@ +!********************************************************************************************************************************** +! LICENSING +! Copyright (C) 2026 National Renewable Energy Laboratory +! +! This file is part of Ambient Wind and Array Effects model for FAST.Farm. +! +! Licensed under the Apache License, Version 2.0 (the "License"); +! you may not use this file except in compliance with the License. +! You may obtain a copy of the License at +! +! http://www.apache.org/licenses/LICENSE-2.0 +! +! Unless required by applicable law or agreed to in writing, software +! distributed under the License is distributed on an "AS IS" BASIS, +! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +! See the License for the specific language governing permissions and +! limitations under the License. +! +!********************************************************************************************************************************** +MODULE AWAE_Timings + + USE NWTC_Library + +#ifdef _OPENMP + USE OMP_LIB +#endif + + IMPLICIT NONE + + integer(IntKi), parameter, public :: AWAE_MaxTimingEntries = 32 + + ! Typically we dissalow the `save` attribute for module variables, but we make an exception here since + ! these variables are used to store timing data that is collected across multiple subroutine calls, but + ! only outside of OMP loops. If we were to allow OMP loops to call these subroutines, then we would + ! need to use threadprivate variables instead of save variables. + integer(IntKi), save :: AWAE_TimingCount = 0 + character(128), save :: AWAE_TimingLabel(AWAE_MaxTimingEntries) + real(DbKi), save :: AWAE_TimingSerial(AWAE_MaxTimingEntries) = 0.0_DbKi + real(DbKi), save :: AWAE_TimingPar(AWAE_MaxTimingEntries) = 0.0_DbKi + character(64), save :: AWAE_TimingPrefix = '' + +CONTAINS + +subroutine AWAE_SetTimingStage(stage) + character(*), intent(in) :: stage + AWAE_TimingPrefix = stage +end subroutine AWAE_SetTimingStage + +subroutine AWAE_ResetTiming() + AWAE_TimingCount = 0 + AWAE_TimingLabel = '' + AWAE_TimingSerial = 0.0_DbKi + AWAE_TimingPar = 0.0_DbKi + AWAE_TimingPrefix = '' +end subroutine AWAE_ResetTiming + +subroutine AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) + integer(IntKi), intent(out) :: numEntries + character(*), intent(out) :: labels(:) + real(DbKi), intent(out) :: serialTimes(:) + real(DbKi), intent(out) :: parTimes(:) + integer(IntKi) :: n, i + + numEntries = 0 + labels = '' + serialTimes = 0.0_DbKi + parTimes = 0.0_DbKi + + n = min(AWAE_TimingCount, min(size(labels), min(size(serialTimes), size(parTimes)))) + numEntries = n + do i = 1, n + labels(i) = AWAE_TimingLabel(i) + serialTimes(i) = AWAE_TimingSerial(i) + parTimes(i) = AWAE_TimingPar(i) + end do +end subroutine AWAE_GetTimingData + +subroutine AWAE_DrainTimingData(numEntries, labels, serialTimes, parTimes) + integer(IntKi), intent(out) :: numEntries + character(*), intent(out) :: labels(:) + real(DbKi), intent(out) :: serialTimes(:) + real(DbKi), intent(out) :: parTimes(:) + + call AWAE_GetTimingData(numEntries, labels, serialTimes, parTimes) + AWAE_TimingCount = 0 + AWAE_TimingLabel = '' + AWAE_TimingSerial = 0.0_DbKi + AWAE_TimingPar = 0.0_DbKi +end subroutine AWAE_DrainTimingData + +real(DbKi) function AWAE_WallTime() +#ifdef _OPENMP + AWAE_WallTime = omp_get_wtime() +#else + call cpu_time(AWAE_WallTime) +#endif +end function AWAE_WallTime + +subroutine AWAE_AddTiming(label, serialDt, parDt) + character(*), intent(in) :: label + real(DbKi), intent(in) :: serialDt + real(DbKi), intent(in) :: parDt + integer(IntKi) :: i + + do i = 1, AWAE_TimingCount + if (trim(AWAE_TimingLabel(i)) == trim(label)) then + AWAE_TimingSerial(i) = AWAE_TimingSerial(i) + serialDt + AWAE_TimingPar(i) = AWAE_TimingPar(i) + parDt + return + end if + end do + + if (AWAE_TimingCount < AWAE_MaxTimingEntries) then + AWAE_TimingCount = AWAE_TimingCount + 1 + AWAE_TimingLabel(AWAE_TimingCount) = label + AWAE_TimingSerial(AWAE_TimingCount) = serialDt + AWAE_TimingPar(AWAE_TimingCount) = parDt + end if +end subroutine AWAE_AddTiming + +subroutine AWAE_AddStageTiming(action, serialStart, parStart) + character(*), intent(in) :: action + real(DbKi), intent(in) :: serialStart + real(DbKi), intent(in) :: parStart + real(DbKi) :: serialEnd, parEnd + character(128) :: label + + if (len_trim(AWAE_TimingPrefix) == 0) return + + call cpu_time(serialEnd) + parEnd = AWAE_WallTime() + label = trim(AWAE_TimingPrefix)//':'//trim(action) + call AWAE_AddTiming(label, serialEnd-serialStart, parEnd-parStart) +end subroutine AWAE_AddStageTiming + +END MODULE AWAE_Timings diff --git a/modules/nwtc-library/src/NWTC_IO.f90 b/modules/nwtc-library/src/NWTC_IO.f90 index 1fbe24a6c..11779b7a1 100644 --- a/modules/nwtc-library/src/NWTC_IO.f90 +++ b/modules/nwtc-library/src/NWTC_IO.f90 @@ -7845,7 +7845,7 @@ RECURSIVE SUBROUTINE WrScr ( InStr ) INTEGER :: MaxLen ! Maximum number of columns to be written to the screen. INTEGER :: NewLineIndx ! The string index where the NewLine character occurs - CHARACTER(10) :: Frm ! Format specifier for the output. + CHARACTER(11) :: Frm ! Format specifier for the output. CHARACTER(Len=:),allocatable :: Str ! The next string to be processed @@ -7872,8 +7872,8 @@ RECURSIVE SUBROUTINE WrScr ( InStr ) Indent = MIN( Indent, MaxLen-2 ) ! at least 2 characters per line MaxLen = MaxLen - Indent IF ( Indent > 0 ) THEN - Frm = '(1X, X,A)' - WRITE (Frm(5:6),'(I2)') Indent + Frm = '(1X, X,A)' + WRITE (Frm(5:7),'(I3)') Indent ELSE Frm = '(1X,A)' END IF diff --git a/modules/nwtc-library/src/SysFlangLinux.f90 b/modules/nwtc-library/src/SysFlangLinux.f90 index 621e40552..880595109 100644 --- a/modules/nwtc-library/src/SysFlangLinux.f90 +++ b/modules/nwtc-library/src/SysFlangLinux.f90 @@ -56,7 +56,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console. Unit 6 causes ADAMS to crash. - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Linux' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysGnuLinux.f90 b/modules/nwtc-library/src/SysGnuLinux.f90 index 0fb51921b..6eddb9b41 100644 --- a/modules/nwtc-library/src/SysGnuLinux.f90 +++ b/modules/nwtc-library/src/SysGnuLinux.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Linux' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysGnuWin.f90 b/modules/nwtc-library/src/SysGnuWin.f90 index 95794cf19..1422989cb 100644 --- a/modules/nwtc-library/src/SysGnuWin.f90 +++ b/modules/nwtc-library/src/SysGnuWin.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Windows' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysIFL.f90 b/modules/nwtc-library/src/SysIFL.f90 index f2ccbe1cd..bf427e8fb 100644 --- a/modules/nwtc-library/src/SysIFL.f90 +++ b/modules/nwtc-library/src/SysIFL.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'Intel Fortran for Linux' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysIVF.f90 b/modules/nwtc-library/src/SysIVF.f90 index d0e722765..41af20386 100644 --- a/modules/nwtc-library/src/SysIVF.f90 +++ b/modules/nwtc-library/src/SysIVF.f90 @@ -57,7 +57,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 7 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .TRUE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}]; Note: NewLine change to ACHAR(10) here on Windows to fix issues with C/Fortran interoperability using WrScr CHARACTER(*), PARAMETER :: OS_Desc = 'Intel Visual Fortran for Windows'! Description of the language/OS diff --git a/modules/nwtc-library/src/SysIVF_Labview.f90 b/modules/nwtc-library/src/SysIVF_Labview.f90 index ac4253900..51a57144f 100644 --- a/modules/nwtc-library/src/SysIVF_Labview.f90 +++ b/modules/nwtc-library/src/SysIVF_Labview.f90 @@ -74,7 +74,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 7 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. diff --git a/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 b/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 index 6b2a847d2..1072ca8e3 100644 --- a/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 +++ b/modules/nwtc-library/src/SysMatlabLinuxGnu.f90 @@ -60,7 +60,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'GNU Fortran for Linux with Matlab' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 b/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 index a31dc15ca..303206a9a 100644 --- a/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 +++ b/modules/nwtc-library/src/SysMatlabLinuxIntel.f90 @@ -60,7 +60,7 @@ MODULE SysSubs INTEGER, PARAMETER :: ConRecL = 120 ! The record length for console output. INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment. CHARACTER(*), PARAMETER :: NewLine = ACHAR(10) ! The delimiter for New Lines [ Windows is CHAR(13)//CHAR(10); MAC is CHAR(13); Unix is CHAR(10) {CHAR(13)=\r is a line feed, CHAR(10)=\n is a new line}] CHARACTER(*), PARAMETER :: OS_Desc = 'Intel Fortran for Linux with Matlab' ! Description of the language/OS diff --git a/modules/nwtc-library/src/SysMatlabWindows.f90 b/modules/nwtc-library/src/SysMatlabWindows.f90 index c462d92df..417a0e9ef 100644 --- a/modules/nwtc-library/src/SysMatlabWindows.f90 +++ b/modules/nwtc-library/src/SysMatlabWindows.f90 @@ -48,7 +48,7 @@ MODULE SysSubs INTEGER, PUBLIC :: CU = 6 ! The I/O unit for the console (Can be changed with SetConsoleUnit subroutine) - INTEGER, PARAMETER :: MaxWrScrLen = 98 ! The maximum number of characters allowed to be written to a line in WrScr + INTEGER, PARAMETER :: MaxWrScrLen = 256 ! The maximum number of characters allowed to be written to a line in WrScr LOGICAL, PARAMETER :: KBInputOK = .FALSE. ! A flag to tell the program that keyboard input is allowed in the environment.