2 * Copyright (C) 2015 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "audio_utils_fifo"
25 #include <audio_utils/clock_nanosleep.h>
26 #include <audio_utils/fifo.h>
27 #include <audio_utils/futex.h>
28 #include <audio_utils/roundup.h>
29 #include <cutils/log.h>
30 #include <utils/Errors.h>
32 audio_utils_fifo_base::audio_utils_fifo_base(uint32_t frameCount,
33 audio_utils_fifo_index& writerRear, audio_utils_fifo_index *throttleFront)
34 __attribute__((no_sanitize("integer"))) :
35 mFrameCount(frameCount), mFrameCountP2(roundup(frameCount)),
36 mFudgeFactor(mFrameCountP2 - mFrameCount),
37 // FIXME need an API to configure the sync types
38 mWriterRear(writerRear), mWriterRearSync(AUDIO_UTILS_FIFO_SYNC_SHARED),
39 mThrottleFront(throttleFront), mThrottleFrontSync(AUDIO_UTILS_FIFO_SYNC_SHARED),
42 // actual upper bound on frameCount will depend on the frame size
43 LOG_ALWAYS_FATAL_IF(frameCount == 0 || frameCount > ((uint32_t) INT32_MAX));
46 audio_utils_fifo_base::~audio_utils_fifo_base()
50 uint32_t audio_utils_fifo_base::sum(uint32_t index, uint32_t increment) const
51 __attribute__((no_sanitize("integer")))
53 if (mFudgeFactor > 0) {
54 uint32_t mask = mFrameCountP2 - 1;
55 ALOG_ASSERT((index & mask) < mFrameCount);
56 ALOG_ASSERT(increment <= mFrameCountP2);
57 if ((index & mask) + increment >= mFrameCount) {
58 increment += mFudgeFactor;
61 ALOG_ASSERT((index & mask) < mFrameCount);
64 return index + increment;
68 int32_t audio_utils_fifo_base::diff(uint32_t rear, uint32_t front, size_t *lost, bool flush) const
69 __attribute__((no_sanitize("integer")))
71 // TODO replace multiple returns by a single return point so this isn't needed
78 uint32_t diff = rear - front;
79 if (mFudgeFactor > 0) {
80 uint32_t mask = mFrameCountP2 - 1;
81 uint32_t rearOffset = rear & mask;
82 uint32_t frontOffset = front & mask;
83 if (rearOffset >= mFrameCount || frontOffset >= mFrameCount) {
84 ALOGE("%s frontOffset=%u rearOffset=%u mFrameCount=%u",
85 __func__, frontOffset, rearOffset, mFrameCount);
89 // genDiff is the difference between the generation count fields of rear and front,
90 // and is always a multiple of mFrameCountP2.
91 uint32_t genDiff = (rear & ~mask) - (front & ~mask);
92 // It's OK for writer to be one generation beyond reader,
93 // but reader has lost frames if writer is further than one generation beyond.
94 if (genDiff > mFrameCountP2) {
96 // Calculate the number of lost frames as the raw difference,
97 // less the mFrameCount frames that are still valid and can be read on retry,
98 // less the wasted indices that don't count as true lost frames.
99 *lost = diff - (flush ? 0 : mFrameCount) - mFudgeFactor * (genDiff/mFrameCountP2);
103 // If writer is one generation beyond reader, skip over the wasted indices.
105 diff -= mFudgeFactor;
106 // Note is still possible for diff > mFrameCount. BCD 16 - BCD 1 shows the problem.
107 // genDiff is 16, fudge is 6, decimal diff is 15 = (22 - 1 - 6).
108 // So we need to check diff for overflow one more time. See "if" a few lines below.
111 // FIFO should not be overfull
112 if (diff > mFrameCount) {
114 *lost = diff - (flush ? 0 : mFrameCount);
118 return (int32_t) diff;
121 void audio_utils_fifo_base::shutdown() const
123 ALOGE("%s", __func__);
127 ////////////////////////////////////////////////////////////////////////////////
129 audio_utils_fifo::audio_utils_fifo(uint32_t frameCount, uint32_t frameSize, void *buffer,
130 audio_utils_fifo_index& writerRear, audio_utils_fifo_index *throttleFront)
131 __attribute__((no_sanitize("integer"))) :
132 audio_utils_fifo_base(frameCount, writerRear, throttleFront),
133 mFrameSize(frameSize), mBuffer(buffer)
135 // maximum value of frameCount * frameSize is INT32_MAX (2^31 - 1), not 2^31, because we need to
136 // be able to distinguish successful and error return values from read and write.
137 LOG_ALWAYS_FATAL_IF(frameCount == 0 || frameSize == 0 || buffer == NULL ||
138 frameCount > ((uint32_t) INT32_MAX) / frameSize);
141 audio_utils_fifo::audio_utils_fifo(uint32_t frameCount, uint32_t frameSize, void *buffer,
142 bool throttlesWriter) :
143 audio_utils_fifo(frameCount, frameSize, buffer, mSingleProcessSharedRear,
144 throttlesWriter ? &mSingleProcessSharedFront : NULL)
148 audio_utils_fifo::~audio_utils_fifo()
152 ////////////////////////////////////////////////////////////////////////////////
154 audio_utils_fifo_provider::audio_utils_fifo_provider(audio_utils_fifo& fifo) :
155 mFifo(fifo), mObtained(0), mTotalReleased(0)
159 audio_utils_fifo_provider::~audio_utils_fifo_provider()
163 ////////////////////////////////////////////////////////////////////////////////
165 audio_utils_fifo_writer::audio_utils_fifo_writer(audio_utils_fifo& fifo) :
166 audio_utils_fifo_provider(fifo), mLocalRear(0),
167 mArmLevel(fifo.mFrameCount), mTriggerLevel(0),
168 mIsArmed(true), // because initial fill level of zero is < mArmLevel
169 mEffectiveFrames(fifo.mFrameCount)
173 audio_utils_fifo_writer::~audio_utils_fifo_writer()
177 ssize_t audio_utils_fifo_writer::write(const void *buffer, size_t count,
178 const struct timespec *timeout)
179 __attribute__((no_sanitize("integer")))
181 audio_utils_iovec iovec[2];
182 ssize_t availToWrite = obtain(iovec, count, timeout);
183 if (availToWrite > 0) {
184 memcpy((char *) mFifo.mBuffer + iovec[0].mOffset * mFifo.mFrameSize, buffer,
185 iovec[0].mLength * mFifo.mFrameSize);
186 if (iovec[1].mLength > 0) {
187 memcpy((char *) mFifo.mBuffer + iovec[1].mOffset * mFifo.mFrameSize,
188 (char *) buffer + (iovec[0].mLength * mFifo.mFrameSize),
189 iovec[1].mLength * mFifo.mFrameSize);
191 release(availToWrite);
196 // iovec == NULL is not part of the public API, but internally it means don't set mObtained
197 ssize_t audio_utils_fifo_writer::obtain(audio_utils_iovec iovec[2], size_t count,
198 const struct timespec *timeout)
199 __attribute__((no_sanitize("integer")))
203 if (mFifo.mThrottleFront != NULL) {
204 int retries = kRetries;
207 front = mFifo.mThrottleFront->loadAcquire();
208 // returns -EIO if mIsShutdown
209 int32_t filled = mFifo.diff(mLocalRear, front);
211 // on error, return an empty slice
216 availToWrite = mEffectiveFrames > (uint32_t) filled ?
217 mEffectiveFrames - (uint32_t) filled : 0;
218 // TODO pull out "count == 0"
219 if (count == 0 || availToWrite > 0 || timeout == NULL ||
220 (timeout->tv_sec == 0 && timeout->tv_nsec == 0)) {
224 // TODO abstract out switch and replace by general sync object
225 // the high level code (synchronization, sleep, futex, iovec) should be completely
226 // separate from the low level code (indexes, available, masking).
228 switch (mFifo.mThrottleFrontSync) {
229 case AUDIO_UTILS_FIFO_SYNC_SLEEP:
230 err = audio_utils_clock_nanosleep(CLOCK_MONOTONIC, 0 /*flags*/, timeout,
233 LOG_ALWAYS_FATAL_IF(errno != EINTR, "unexpected err=%d errno=%d", err, errno);
239 case AUDIO_UTILS_FIFO_SYNC_PRIVATE:
240 op = FUTEX_WAIT_PRIVATE;
242 case AUDIO_UTILS_FIFO_SYNC_SHARED:
243 if (timeout->tv_sec == LONG_MAX) {
246 err = mFifo.mThrottleFront->wait(op, front, timeout);
250 // Benign race condition with partner: mFifo.mThrottleFront->mIndex
251 // changed value between the earlier atomic_load_explicit() and sys_futex().
252 // Try to load index again, but give up if we are unable to converge.
254 // bypass the "timeout = NULL;" below
263 LOG_ALWAYS_FATAL("unexpected err=%d errno=%d", err, errno);
269 LOG_ALWAYS_FATAL("mFifo.mThrottleFrontSync=%d", mFifo.mThrottleFrontSync);
275 if (mFifo.mIsShutdown) {
279 availToWrite = mEffectiveFrames;
282 if (availToWrite > count) {
283 availToWrite = count;
285 uint32_t rearOffset = mLocalRear & (mFifo.mFrameCountP2 - 1);
286 size_t part1 = mFifo.mFrameCount - rearOffset;
287 if (part1 > availToWrite) {
288 part1 = availToWrite;
290 size_t part2 = part1 > 0 ? availToWrite - part1 : 0;
293 iovec[0].mOffset = rearOffset;
294 iovec[0].mLength = part1;
295 iovec[1].mOffset = 0;
296 iovec[1].mLength = part2;
297 mObtained = availToWrite;
299 return availToWrite > 0 ? availToWrite : err;
302 void audio_utils_fifo_writer::release(size_t count)
303 __attribute__((no_sanitize("integer")))
305 // no need to do an early check for mIsShutdown, because the extra code executed is harmless
307 if (count > mObtained) {
308 ALOGE("%s(count=%zu) > mObtained=%u", __func__, count, mObtained);
312 if (mFifo.mThrottleFront != NULL) {
313 uint32_t front = mFifo.mThrottleFront->loadAcquire();
314 // returns -EIO if mIsShutdown
315 int32_t filled = mFifo.diff(mLocalRear, front);
316 mLocalRear = mFifo.sum(mLocalRear, count);
317 mFifo.mWriterRear.storeRelease(mLocalRear);
320 switch (mFifo.mWriterRearSync) {
321 case AUDIO_UTILS_FIFO_SYNC_SLEEP:
323 case AUDIO_UTILS_FIFO_SYNC_PRIVATE:
324 op = FUTEX_WAKE_PRIVATE;
326 case AUDIO_UTILS_FIFO_SYNC_SHARED:
328 if ((uint32_t) filled < mArmLevel) {
331 if (mIsArmed && filled + count > mTriggerLevel) {
332 int err = mFifo.mWriterRear.wake(op, INT32_MAX /*waiters*/);
333 // err is number of processes woken up
335 LOG_ALWAYS_FATAL("%s: unexpected err=%d errno=%d",
336 __func__, err, errno);
343 LOG_ALWAYS_FATAL("mFifo.mWriterRearSync=%d", mFifo.mWriterRearSync);
347 mLocalRear = mFifo.sum(mLocalRear, count);
348 mFifo.mWriterRear.storeRelease(mLocalRear);
351 mTotalReleased += count;
355 ssize_t audio_utils_fifo_writer::available()
357 // iovec == NULL is not part of the public API, but internally it means don't set mObtained
358 return obtain(NULL /*iovec*/, SIZE_MAX /*count*/, NULL /*timeout*/);
361 void audio_utils_fifo_writer::resize(uint32_t frameCount)
363 // cap to range [0, mFifo.mFrameCount]
364 if (frameCount > mFifo.mFrameCount) {
365 frameCount = mFifo.mFrameCount;
367 // if we reduce the effective frame count, update hysteresis points to be within the new range
368 if (frameCount < mEffectiveFrames) {
369 if (mArmLevel > frameCount) {
370 mArmLevel = frameCount;
372 if (mTriggerLevel > frameCount) {
373 mTriggerLevel = frameCount;
376 mEffectiveFrames = frameCount;
379 uint32_t audio_utils_fifo_writer::size() const
381 return mEffectiveFrames;
384 void audio_utils_fifo_writer::setHysteresis(uint32_t lowLevelArm, uint32_t highLevelTrigger)
386 // cap to range [0, mEffectiveFrames]
387 if (lowLevelArm > mEffectiveFrames) {
388 lowLevelArm = mEffectiveFrames;
390 if (highLevelTrigger > mEffectiveFrames) {
391 highLevelTrigger = mEffectiveFrames;
393 // TODO this is overly conservative; it would be better to arm based on actual fill level
394 if (lowLevelArm > mArmLevel) {
397 mArmLevel = lowLevelArm;
398 mTriggerLevel = highLevelTrigger;
401 void audio_utils_fifo_writer::getHysteresis(uint32_t *armLevel, uint32_t *triggerLevel) const
403 *armLevel = mArmLevel;
404 *triggerLevel = mTriggerLevel;
407 ////////////////////////////////////////////////////////////////////////////////
409 audio_utils_fifo_reader::audio_utils_fifo_reader(audio_utils_fifo& fifo, bool throttlesWriter,
411 audio_utils_fifo_provider(fifo),
413 // If we throttle the writer, then initialize our front index to zero so that we see all data
414 // currently in the buffer.
415 // Otherwise, ignore everything currently in the buffer by initializing our front index to the
416 // current value of writer's rear. This avoids an immediate -EOVERFLOW (overrun) in the case
417 // where reader starts out more than one buffer behind writer. The initial catch-up does not
418 // contribute towards the totalLost, totalFlushed, or totalReleased counters.
419 mLocalFront(throttlesWriter ? 0 : mFifo.mWriterRear.loadConsume()),
421 mThrottleFront(throttlesWriter ? mFifo.mThrottleFront : NULL),
423 mArmLevel(-1), mTriggerLevel(mFifo.mFrameCount),
424 mIsArmed(true), // because initial fill level of zero is > mArmLevel
425 mTotalLost(0), mTotalFlushed(0)
429 audio_utils_fifo_reader::~audio_utils_fifo_reader()
431 // TODO Need a way to pass throttle capability to the another reader, should one reader exit.
434 ssize_t audio_utils_fifo_reader::read(void *buffer, size_t count, const struct timespec *timeout,
436 __attribute__((no_sanitize("integer")))
438 audio_utils_iovec iovec[2];
439 ssize_t availToRead = obtain(iovec, count, timeout, lost);
440 if (availToRead > 0) {
441 memcpy(buffer, (char *) mFifo.mBuffer + iovec[0].mOffset * mFifo.mFrameSize,
442 iovec[0].mLength * mFifo.mFrameSize);
443 if (iovec[1].mLength > 0) {
444 memcpy((char *) buffer + (iovec[0].mLength * mFifo.mFrameSize),
445 (char *) mFifo.mBuffer + iovec[1].mOffset * mFifo.mFrameSize,
446 iovec[1].mLength * mFifo.mFrameSize);
448 release(availToRead);
453 ssize_t audio_utils_fifo_reader::obtain(audio_utils_iovec iovec[2], size_t count,
454 const struct timespec *timeout)
455 __attribute__((no_sanitize("integer")))
457 return obtain(iovec, count, timeout, NULL /*lost*/);
460 void audio_utils_fifo_reader::release(size_t count)
461 __attribute__((no_sanitize("integer")))
463 // no need to do an early check for mIsShutdown, because the extra code executed is harmless
465 if (count > mObtained) {
466 ALOGE("%s(count=%zu) > mObtained=%u", __func__, count, mObtained);
470 if (mThrottleFront != NULL) {
471 uint32_t rear = mFifo.mWriterRear.loadAcquire();
472 // returns -EIO if mIsShutdown
473 int32_t filled = mFifo.diff(rear, mLocalFront);
474 mLocalFront = mFifo.sum(mLocalFront, count);
475 mThrottleFront->storeRelease(mLocalFront);
478 switch (mFifo.mThrottleFrontSync) {
479 case AUDIO_UTILS_FIFO_SYNC_SLEEP:
481 case AUDIO_UTILS_FIFO_SYNC_PRIVATE:
482 op = FUTEX_WAKE_PRIVATE;
484 case AUDIO_UTILS_FIFO_SYNC_SHARED:
486 if (filled > mArmLevel) {
489 if (mIsArmed && filled - count < mTriggerLevel) {
490 int err = mThrottleFront->wake(op, 1 /*waiters*/);
491 // err is number of processes woken up
492 if (err < 0 || err > 1) {
493 LOG_ALWAYS_FATAL("%s: unexpected err=%d errno=%d",
494 __func__, err, errno);
501 LOG_ALWAYS_FATAL("mFifo.mThrottleFrontSync=%d", mFifo.mThrottleFrontSync);
505 mLocalFront = mFifo.sum(mLocalFront, count);
508 mTotalReleased += count;
512 // iovec == NULL is not part of the public API, but internally it means don't set mObtained
513 ssize_t audio_utils_fifo_reader::obtain(audio_utils_iovec iovec[2], size_t count,
514 const struct timespec *timeout, size_t *lost)
515 __attribute__((no_sanitize("integer")))
518 int retries = kRetries;
521 rear = mFifo.mWriterRear.loadAcquire();
522 // TODO pull out "count == 0"
523 if (count == 0 || rear != mLocalFront || timeout == NULL ||
524 (timeout->tv_sec == 0 && timeout->tv_nsec == 0)) {
529 switch (mFifo.mWriterRearSync) {
530 case AUDIO_UTILS_FIFO_SYNC_SLEEP:
531 err = audio_utils_clock_nanosleep(CLOCK_MONOTONIC, 0 /*flags*/, timeout,
534 LOG_ALWAYS_FATAL_IF(errno != EINTR, "unexpected err=%d errno=%d", err, errno);
540 case AUDIO_UTILS_FIFO_SYNC_PRIVATE:
541 op = FUTEX_WAIT_PRIVATE;
543 case AUDIO_UTILS_FIFO_SYNC_SHARED:
544 if (timeout->tv_sec == LONG_MAX) {
547 err = mFifo.mWriterRear.wait(op, rear, timeout);
551 // Benign race condition with partner: mFifo.mWriterRear->mIndex
552 // changed value between the earlier atomic_load_explicit() and sys_futex().
553 // Try to load index again, but give up if we are unable to converge.
555 // bypass the "timeout = NULL;" below
564 LOG_ALWAYS_FATAL("unexpected err=%d errno=%d", err, errno);
570 LOG_ALWAYS_FATAL("mFifo.mWriterRearSync=%d", mFifo.mWriterRearSync);
579 // returns -EIO if mIsShutdown
580 int32_t filled = mFifo.diff(rear, mLocalFront, lost, mFlush);
582 mTotalReleased += *lost;
584 if (filled == -EOVERFLOW) {
585 // catch up with writer, but preserve the still valid frames in buffer
586 mLocalFront = rear - (mFlush ? 0 : mFifo.mFrameCountP2 /*sic*/);
588 // on error, return an empty slice
592 size_t availToRead = (size_t) filled;
593 if (availToRead > count) {
596 uint32_t frontOffset = mLocalFront & (mFifo.mFrameCountP2 - 1);
597 size_t part1 = mFifo.mFrameCount - frontOffset;
598 if (part1 > availToRead) {
601 size_t part2 = part1 > 0 ? availToRead - part1 : 0;
604 iovec[0].mOffset = frontOffset;
605 iovec[0].mLength = part1;
606 iovec[1].mOffset = 0;
607 iovec[1].mLength = part2;
608 mObtained = availToRead;
610 return availToRead > 0 ? availToRead : err;
613 ssize_t audio_utils_fifo_reader::available()
615 return available(NULL /*lost*/);
618 ssize_t audio_utils_fifo_reader::available(size_t *lost)
620 // iovec == NULL is not part of the public API, but internally it means don't set mObtained
621 return obtain(NULL /*iovec*/, SIZE_MAX /*count*/, NULL /*timeout*/, lost);
624 ssize_t audio_utils_fifo_reader::flush(size_t *lost)
626 audio_utils_iovec iovec[2];
627 ssize_t ret = obtain(iovec, SIZE_MAX /*count*/, NULL /*timeout*/, lost);
629 size_t flushed = (size_t) ret;
631 mTotalFlushed += flushed;
637 void audio_utils_fifo_reader::setHysteresis(int32_t armLevel, uint32_t triggerLevel)
639 // cap to range [0, mFifo.mFrameCount]
642 } else if ((uint32_t) armLevel > mFifo.mFrameCount) {
643 armLevel = mFifo.mFrameCount;
645 if (triggerLevel > mFifo.mFrameCount) {
646 triggerLevel = mFifo.mFrameCount;
648 // TODO this is overly conservative; it would be better to arm based on actual fill level
649 if (armLevel < mArmLevel) {
652 mArmLevel = armLevel;
653 mTriggerLevel = triggerLevel;
656 void audio_utils_fifo_reader::getHysteresis(int32_t *armLevel, uint32_t *triggerLevel) const
658 *armLevel = mArmLevel;
659 *triggerLevel = mTriggerLevel;