Skip to content

Commit 52ee93f

Browse files
4eUeP4eUeP
committed
tmp
1 parent de0d162 commit 52ee93f

File tree

5 files changed

+133
-85
lines changed

5 files changed

+133
-85
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 116 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ jobs:
5353
build-hstream-admin-store:
5454
needs: pre-build
5555
runs-on: ubuntu-latest
56+
if: false
5657
strategy:
5758
fail-fast: false
5859
steps:
@@ -159,97 +160,128 @@ jobs:
159160
${{ env.CABAL }} update
160161
${{ env.SHELL }} "'make clean'"
161162
${{ env.SHELL }} make
162-
${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} build --enable-tests --enable-benchmarks all
163-
${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} install hstream
164-
165-
- name: start hstream server
166-
run: |
167-
export CONTAINER_NAME=$TEST_CONTAINER_NAME
168-
export IMAGE="docker.io/hstreamdb/haskell:${{ matrix.ghc }}"
169-
export EXTRA_OPTS="--check --no-interactive --detach"
170-
export COMMAND=" "
171-
export EXE=$(find dist-newstyle -name "hstream-server" -type f)
172-
./script/start-server.sh
173-
sleep 5
174-
docker logs --tail 100 $TEST_CONTAINER_NAME
175-
176-
- name: test
177-
run: |
178-
${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} test --test-show-details=direct all
163+
${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} build hstream-common-stats-test
164+
# ${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} build --enable-tests --enable-benchmarks all
165+
# ${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} install hstream
166+
167+
# - name: start hstream server
168+
# run: |
169+
# export CONTAINER_NAME=$TEST_CONTAINER_NAME
170+
# export IMAGE="docker.io/hstreamdb/haskell:${{ matrix.ghc }}"
171+
# export EXTRA_OPTS="--check --no-interactive --detach"
172+
# export COMMAND=" "
173+
# export EXE=$(find dist-newstyle -name "hstream-server" -type f)
174+
# ./script/start-server.sh
175+
# sleep 5
176+
# docker logs --tail 100 $TEST_CONTAINER_NAME
177+
#
178+
# - name: test
179+
# run: |
180+
# for i in {1..100}; do
181+
# ${{ env.CABAL }} ${{ env.EXTRA_CABAL_ARGS }} test --test-show-details=direct all
182+
# done
183+
#
184+
# - name: collect hserver logs
185+
# if: ${{ success() }} || ${{ failure() }}
186+
# run: |
187+
# rm -rf hserver.log
188+
# docker logs $TEST_CONTAINER_NAME &> hserver.log
189+
#
190+
# - name: upload hserver logs
191+
# uses: actions/upload-artifact@v3
192+
# if: ${{ success() }} || ${{ failure() }}
193+
# with:
194+
# name: hserver-logs-${{ matrix.ghc }}
195+
# path: hserver.log
196+
# retention-days: 7
197+
#
198+
# # Due to an [cabal bug](https://github.com/haskell/cabal/issues/7423),
199+
# # `cabal check` will emit a warning even if the `-O2` option is just
200+
# # an flag. This is disabled until the problem is fixed.
201+
# #- name: check
202+
# # run: |
203+
# # python3 script/dev-tools cabal --check --no-interactive -i docker.io/hstreamdb/haskell:${{ matrix.ghc }} -- sdist all
204+
#
205+
# # # unfortunately, there is no `cabal check all`
206+
# # #log_info "Run all cabal check..."
207+
# # # Note that we ignore hstream-store package to run cabal check, because there
208+
# # # is an unexpected warning:
209+
# # # ...
210+
# # # Warning: 'cpp-options': -std=c++17 is not portable C-preprocessor flag
211+
# # # Warning: Hackage would reject this package.
212+
# # for dir in hstream-sql hstream-processing hstream; do
213+
# # python3 script/dev-tools shell --check --no-interactive -i docker.io/hstreamdb/haskell:${{ matrix.ghc }} "'cd $dir && cabal check'"
214+
# # done
215+
#
216+
# # TODO
217+
# #- name: haddock
218+
# # python3 script/dev-tools cabal --check --no-interactive -i docker.io/hstreamdb/haskell:${{ matrix.ghc }} -- haddock haddock --enable-documentation --haddock-for-hackage all
219+
#
220+
# # -------------------------------------------------------------------------------
221+
#
222+
# - name: stop all started services
223+
# run: docker rm -f $(docker ps -a -q)
224+
#
225+
# # NOTE: The quick-build-dev-image relies on the "hstreamdb/hstream" base image.
226+
# # If you have installed any additional libraries in the builder image (hstreamdb/haskell),
227+
# # and these libraries are required (e.g., if a lib.so file is needed), you may encounter a
228+
# # linking error during the integration tests that follow. In such cases, you will need to
229+
# # publish a new version of the hstreamdb/hstream image first, which includes the necessary
230+
# # libraries.
231+
# - name: quick build new hstream image
232+
# run: |
233+
# mkdir -p ~/data
234+
# if [ "${{ env.GHC_MAJOR_VER }}" = "8" ]; then
235+
# python3 script/dev-tools quick-build-dev-image \
236+
# --builder-image docker.io/hstreamdb/haskell:${{ matrix.ghc }} \
237+
# --project-file cabal.project.ghc810 \
238+
# --only-hstream \
239+
# -t $NEW_HSTREAM_IMAGE
240+
# else
241+
# python3 script/dev-tools quick-build-dev-image \
242+
# --builder-image docker.io/hstreamdb/haskell:${{ matrix.ghc }} \
243+
# --only-hstream \
244+
# -t $NEW_HSTREAM_IMAGE
245+
# fi
246+
#
247+
# docker save -o ~/data/new_hstream_image.tar $NEW_HSTREAM_IMAGE
248+
#
249+
# - uses: actions/upload-artifact@v3
250+
# with:
251+
# name: image-testing-${{ matrix.ghc }}
252+
# path: ~/data/new_hstream_image.tar
253+
# retention-days: 2
179254

180-
- name: collect hserver logs
181-
if: ${{ success() }} || ${{ failure() }}
182-
run: |
183-
rm -rf hserver.log
184-
docker logs $TEST_CONTAINER_NAME &> hserver.log
255+
- uses: actions/upload-artifact@v3
256+
with:
257+
name: stats_test
258+
path: dist-newstyle/build/x86_64-linux/ghc-9.2.7/hstream-common-stats-0.1.0.0/build/hstream-common-stats-test/hstream-common-stats-test
259+
retention-days: 2
185260

186-
- name: upload hserver logs
187-
uses: actions/upload-artifact@v3
188-
if: ${{ success() }} || ${{ failure() }}
261+
stats-tests:
262+
needs: [build-and-test]
263+
runs-on: ubuntu-latest
264+
name: run-stats-tests
265+
steps:
266+
- uses: actions/download-artifact@v3
189267
with:
190-
name: hserver-logs-${{ matrix.ghc }}
191-
path: hserver.log
192-
retention-days: 7
268+
name: stats_test
269+
path: ~/data/hstream-common-stats-test
193270

194-
# Due to an [cabal bug](https://github.com/haskell/cabal/issues/7423),
195-
# `cabal check` will emit a warning even if the `-O2` option is just
196-
# an flag. This is disabled until the problem is fixed.
197-
#- name: check
198-
# run: |
199-
# python3 script/dev-tools cabal --check --no-interactive -i docker.io/hstreamdb/haskell:${{ matrix.ghc }} -- sdist all
200-
201-
# # unfortunately, there is no `cabal check all`
202-
# #log_info "Run all cabal check..."
203-
# # Note that we ignore hstream-store package to run cabal check, because there
204-
# # is an unexpected warning:
205-
# # ...
206-
# # Warning: 'cpp-options': -std=c++17 is not portable C-preprocessor flag
207-
# # Warning: Hackage would reject this package.
208-
# for dir in hstream-sql hstream-processing hstream; do
209-
# python3 script/dev-tools shell --check --no-interactive -i docker.io/hstreamdb/haskell:${{ matrix.ghc }} "'cd $dir && cabal check'"
210-
# done
211-
212-
# TODO
213-
#- name: haddock
214-
# python3 script/dev-tools cabal --check --no-interactive -i docker.io/hstreamdb/haskell:${{ matrix.ghc }} -- haddock haddock --enable-documentation --haddock-for-hackage all
215-
216-
# -------------------------------------------------------------------------------
217-
218-
- name: stop all started services
219-
run: docker rm -f $(docker ps -a -q)
220-
221-
# NOTE: The quick-build-dev-image relies on the "hstreamdb/hstream" base image.
222-
# If you have installed any additional libraries in the builder image (hstreamdb/haskell),
223-
# and these libraries are required (e.g., if a lib.so file is needed), you may encounter a
224-
# linking error during the integration tests that follow. In such cases, you will need to
225-
# publish a new version of the hstreamdb/hstream image first, which includes the necessary
226-
# libraries.
227-
- name: quick build new hstream image
271+
- name: CPU info
228272
run: |
229-
mkdir -p ~/data
230-
if [ "${{ env.GHC_MAJOR_VER }}" = "8" ]; then
231-
python3 script/dev-tools quick-build-dev-image \
232-
--builder-image docker.io/hstreamdb/haskell:${{ matrix.ghc }} \
233-
--project-file cabal.project.ghc810 \
234-
--only-hstream \
235-
-t $NEW_HSTREAM_IMAGE
236-
else
237-
python3 script/dev-tools quick-build-dev-image \
238-
--builder-image docker.io/hstreamdb/haskell:${{ matrix.ghc }} \
239-
--only-hstream \
240-
-t $NEW_HSTREAM_IMAGE
241-
fi
273+
sudo apt-get install cpuid
274+
cpuid
242275
243-
docker save -o ~/data/new_hstream_image.tar $NEW_HSTREAM_IMAGE
244-
245-
- uses: actions/upload-artifact@v3
246-
with:
247-
name: image-testing-${{ matrix.ghc }}
248-
path: ~/data/new_hstream_image.tar
249-
retention-days: 2
276+
- run: |
277+
chmod +x ~/data/hstream-common-stats-test/hstream-common-stats-test
278+
for i in {1..1000}; do
279+
docker run -t --rm -v $HOME/data:/data docker.io/hstreamdb/haskell:9.2.7 /data/hstream-common-stats-test/hstream-common-stats-test
280+
done
250281
251282
integration-tests:
252283
needs: [pre-build, build-and-test]
284+
if: false
253285
runs-on: ubuntu-latest
254286
name: integration-tests-ghc-${{ matrix.ghc }}
255287
strategy:
@@ -307,6 +339,7 @@ jobs:
307339

308340
integration-tests-rqlite:
309341
needs: [pre-build, build-and-test]
342+
if: false
310343
runs-on: ubuntu-latest
311344
name: integration-tests-rqlite-ghc-${{ matrix.ghc }}
312345
strategy:
@@ -365,6 +398,7 @@ jobs:
365398

366399
distributed-tests:
367400
needs: [pre-build, build-and-test]
401+
if: false
368402
runs-on: ubuntu-latest
369403
name: distributed-tests-ghc-${{ matrix.ghc }}
370404
strategy:

‎common/stats/cbits/stats/Stats.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <folly/small_vector.h>
88
#include <folly/stats/BucketedTimeSeries.h>
99
#include <folly/stats/MultiLevelTimeSeries.h>
10+
#include <iostream>
1011
#include <logdevice/common/UpdateableSharedPtr.h>
1112
#include <logdevice/common/checks.h>
1213
#include <logdevice/common/stats/Stats.h>
@@ -59,6 +60,7 @@ template <typename VT, typename CT>
5960
void MultiLevelTimeSeriesWrapper<VT, CT>::addValue(const ValueType& n) {
6061
auto now = std::chrono::duration_cast<Duration>(
6162
std::chrono::steady_clock::now().time_since_epoch());
63+
std::cout << "-> Add value: " << now.count() << std::endl;
6264
folly::MultiLevelTimeSeries<VT, CT>::addValue(now, n);
6365
}
6466

@@ -523,6 +525,7 @@ int perXTimeSeriesGet(
523525
folly::Synchronized<Map> Stats::*stats_member_map,
524526
//
525527
HsInt interval_size, HsInt* ms_intervals, HsDouble* aggregate_vals) {
528+
printf("-> key: %s\n", key);
526529
using Duration = typename PerXTimeSeries::Duration;
527530
using TimePoint = typename PerXTimeSeries::TimePoint;
528531

@@ -535,12 +538,14 @@ int perXTimeSeriesGet(
535538

536539
bool has_found = false;
537540
stats_holder->runForEach([&](Stats& s) {
541+
printf("-> each stats\n");
538542
// Use synchronizedCopy() so we do not have to hold a read lock on
539543
// per_x_stats map while we iterate over individual entries.
540544
for (auto& entry : s.synchronizedCopy(stats_member_map)) {
541545
std::lock_guard<std::mutex> guard(entry.second->mutex);
542546

543547
std::string& key_ = entry.first;
548+
printf("-> key_: %s\n", key_.c_str());
544549
auto time_series = entry.second.get()->*member_ptr;
545550
if (!time_series) {
546551
continue;
@@ -572,6 +577,7 @@ int perXTimeSeriesGet(
572577
// Duration may not be seconds, convert to seconds
573578
aggregate_vals[i] += rate_per_time_type * Duration::period::den /
574579
Duration::period::num;
580+
printf("-> aggregate_vals %d:%f\n", i, aggregate_vals[i]);
575581
}
576582

577583
// We have aggregated the stat from this Stats, because stream name

‎common/stats/test/HStream/StatsSpec.hs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ spec = do
1818
runIO setupFatalSignalHandler
1919

2020
statsSpec
21-
threadedStatsSpec
21+
--threadedStatsSpec
2222

2323
miscSpec
2424

‎common/stats/test/HStream/StatsSpecUtils.hs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import Data.Maybe (fromJust)
77
import Test.Hspec
88
import Z.Data.CBytes (CBytes)
99

10-
import HStream.Stats (StatsHolder, resetStatsHolder)
10+
import HStream.Stats
1111

1212
{-# ANN module ("HLint: ignore Use head" :: String) #-}
1313

@@ -20,6 +20,7 @@ mkTimeSeriesTest
2020
-> (StatsHolder -> CBytes -> [Int] -> IO (Either String (Map.Map CBytes [Double])))
2121
-> Expectation
2222
mkTimeSeriesTest h intervals stats_name stats_add stats_get stats_getall = do
23+
print "================== start ================"
2324
stats_add h "key_1" 1000
2425
stats_add h "key_2" 10000
2526
-- NOTE: we choose to sleep 1s so that we can assume the speed of key_1
@@ -28,18 +29,25 @@ mkTimeSeriesTest h intervals stats_name stats_add stats_get stats_getall = do
2829
stats_add h "key_1" 1000
2930
stats_add h "key_2" 10000
3031

32+
printStatsHolder h
33+
print $ "=> stats_get " <> show stats_name <> " " <> show intervals
34+
35+
print "=> non-existed-key"
3136
stats_get h stats_name "non-existed-key" intervals
3237
`shouldReturn` Nothing
3338

39+
print "=> key1"
3440
Just [rate1_p5s, rate1_p10s] <- stats_get h stats_name "key_1" intervals
3541
rate1_p5s `shouldSatisfy` (\s -> s > 0 && s <= 2000)
3642
rate1_p10s `shouldSatisfy` (\s -> s > 0 && s <= 2000)
43+
print "=> key2"
3744
Just [rate2_p5s, rate2_p10s] <- stats_get h stats_name "key_2" intervals
3845
-- NOTE: There is a possibility that the speed is less than 2000. However, in
3946
-- typical cases, it shouldn't.
4047
rate2_p5s `shouldSatisfy` (\s -> s > 2000 && s <= 20000)
4148
rate2_p10s `shouldSatisfy` (\s -> s > 2000 && s <= 20000)
4249

50+
print "================== end ================"
4351
Right m <- stats_getall h stats_name intervals
4452
Map.lookup "key_1" m `shouldSatisfy` ((\s -> s!!0 > 0 && s!!0 <= 2000) . fromJust)
4553
Map.lookup "key_2" m `shouldSatisfy` ((\s -> s!!1 > 2000 && s!!1 <= 20000) . fromJust)

‎external/proto3-suite‎

0 commit comments

Comments
 (0)